iop/agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py
toki 8a4f6c55a1 sync: roadmap, skills, test inventory, streamgate package, docs updates
- Update roadmap milestones and phase docs across multiple phases
- Update plan, code-review, create-roadmap, update-roadmap, finalize-task-routing skills
- Update dev-corp-runtime-deploy, dev-runtime-deploy, orchestrate-agent-task-loop skills
- Refactor agent-task-loop dispatch script
- Add streamgate Go package (commit_boundary, evidence_tail, filter_registry, stream_release)
- Add test inventory files (dev, dev-corp, unified)
- Update test smoke tests and rules for dev/dev-corp
- Update docs/edge-local-dev-guide and e2e scripts
- Update inventory-query Go package
- Remove deprecated templates and inventory.yaml files
- Add orchestrate-agent-task-loop tests
2026-07-25 11:41:08 +09:00

3152 lines
122 KiB
Python

#!/usr/bin/env python3
"""Dispatch every independently-ready agent-task pair until the task group completes."""
from __future__ import annotations
import argparse
import asyncio
import fcntl
import hashlib
import json
import os
import re
import signal
import shutil
import subprocess
import sys
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
SEP = "-" * 42
PLAN_RE = re.compile(r"^PLAN-(local|cloud)-G(0[1-9]|10)\.md$")
REVIEW_RE = re.compile(r"^CODE_REVIEW-(local|cloud)-G(0[1-9]|10)\.md$")
SUBTASK_RE = re.compile(r"^(?P<index>\d{2})(?:\+(?P<deps>\d{2}(?:,\d{2})*))?_[a-z0-9_]+$")
VERDICT_HEADING_RE = re.compile(r"^## 코드리뷰 결과[ \t]*$", re.MULTILINE)
VERDICT_LINE_RE = re.compile(
r"^(?:-\s*)?(?:\*\*)?종합 판정(?:\*\*)?\s*:\s*(PASS|WARN|FAIL)[ \t]*$",
re.MULTILINE,
)
VERDICT_BLOCK_RE = re.compile(
r"^###\s+종합 판정[ \t]*$\s*^(?:\*\*)?(PASS|WARN|FAIL)(?:\*\*)?[ \t]*$",
re.MULTILINE,
)
PLAN_IDENTITY_RE = re.compile(
r"<!--\s+task=(?P<task>\S+)\s+plan=(?P<plan>\d+)\s+tag=(?P<tag>\S+)\s+-->"
)
FINAL_CHECK_RE = re.compile(r"^-\s+\[[xX]\]\s+CODE_REVIEW-\*-G\?\?\.md", re.MULTILINE)
WORK_LOG_NAME = "WORK_LOG.md"
KST = timezone(timedelta(hours=9), name="KST")
STREAM_HEARTBEAT_SECONDS = 30
PI_SESSION_START_STALL_SECONDS = 60
PI_MODEL_RESPONSE_STALL_SECONDS = 3 * 60
PI_TOOL_STALL_SECONDS = 30 * 60
PI_SESSION_SCHEMA_VERSION = 3
RECOVERY_FAILURE_LIMIT = 10
SELF_CHECK_INCOMPLETE_LIMIT = 10
REVIEW_NO_PROGRESS_LIMIT = 10
PROVIDER_TRANSPORT_FAILURES = frozenset(
{"provider-connection", "provider-stream-disconnect"}
)
FAILURE_EVIDENCE_LIMIT = 2000
# Used only to reject a stale locator whose dispatcher and agent PIDs are both
# gone. A live process is inspected after silence; it is never killed solely by
# this fallback clock.
CODEX_STREAM_STALL_SECONDS = 5 * 60
PROMOTABLE_PATTERNS = {
"context-limit": [
r"context (?:length|window)", r"maximum context", r"prompt is too long",
r"too many tokens", r"token limit", r"exceeded.{0,40}token",
r"output (?:token )?limit", r"maximum output", r"\bmax_tokens\b",
r"response (?:is )?too long",
],
"provider-quota": [
r"rate.?limit", r"\bquota\b", r"resource_exhausted", r"\b429\b",
r"usage limit", r"capacity limit",
],
"model-unavailable": [
r"model.{0,40}(?:not found|unavailable)", r"overloaded",
r"temporarily unavailable",
],
"provider-connection": [
r"\bprovider[_ -]?tunnel[_ -]?error\b",
(
r"(?:provider|backend|/v1/chat/completions|/v1/responses)"
r".{0,160}(?:connection refused|dial tcp)"
),
],
"provider-stream-disconnect": [
r"backend connection failed during streaming request",
r"sse stream before done",
r"llama-server was unresponsive",
r"backend watchdog",
r"model will be reloaded automatically on retry",
(
r"(?:provider|backend|sse).{0,160}"
r"curl error: failure when receiving data from the peer"
),
],
}
def now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def work_log_now_iso() -> str:
return datetime.now(KST).isoformat()
def banner(event: str, task: str, lines: list[str] | None = None) -> None:
display_task = task.rsplit("/", 1)[-1]
print(SEP, flush=True)
print(f"{event}: {display_task}", flush=True)
print(SEP, flush=True)
if display_task != task:
print(f"task={task}", flush=True)
for line in lines or []:
print(line, flush=True)
def sha256_file(path: Path | None) -> str:
if path is None or not path.exists():
return "none"
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(65536), b""):
digest.update(chunk)
return digest.hexdigest()
DISPATCHER_SOURCE_PATH = Path(__file__).resolve()
DISPATCHER_SOURCE_SHA256 = sha256_file(DISPATCHER_SOURCE_PATH)
DISPATCHER_PROCESS_STARTED_AT = now_iso()
def dispatcher_source_provenance() -> dict[str, Any]:
current_sha256 = sha256_file(DISPATCHER_SOURCE_PATH)
return {
"dispatcher_pid": os.getpid(),
"dispatcher_process_started_at": DISPATCHER_PROCESS_STARTED_AT,
"dispatcher_source_path": str(DISPATCHER_SOURCE_PATH),
"dispatcher_source_sha256": DISPATCHER_SOURCE_SHA256,
"dispatcher_source_current_sha256": current_sha256,
"dispatcher_source_matches_loaded": current_sha256 == DISPATCHER_SOURCE_SHA256,
}
def plan_identity(path: Path | None) -> str:
if path is None or not path.exists():
return "none"
text = path.read_text(encoding="utf-8", errors="replace")[:1024]
match = PLAN_IDENTITY_RE.search(text)
if not match:
return sha256_file(path)
identity = "\0".join(match.group(name) for name in ("task", "plan", "tag"))
return "meta:" + hashlib.sha256(identity.encode()).hexdigest()
def write_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
def milestone_work_log_path(task: Task) -> Path:
return (
task.directory.parent / WORK_LOG_NAME
if "/" in task.name
else task.directory / WORK_LOG_NAME
)
def append_milestone_event(
task: Task,
*,
event: str,
execution_id: str,
role: str,
attempt: int,
model: str,
result: str,
locator: Path,
) -> Path:
path = milestone_work_log_path(task)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a+", encoding="utf-8") as stream:
fcntl.flock(stream.fileno(), fcntl.LOCK_EX)
try:
stream.seek(0)
text = stream.read()
if not text:
stream.write(
"# Milestone Work Log\n\n"
"> Dispatcher-owned execution timeline. Workers and reviewers do not edit this file.\n\n"
"| seq | time | event | task | role | attempt | model | result | locator |\n"
"|---:|---|---|---|---|---:|---|---|---|\n"
)
sequence = 1
else:
stream.seek(0, os.SEEK_END)
if "| seq | time | event | task | role | attempt | model | result | locator |" not in text:
if not text.endswith("\n"):
stream.write("\n")
stream.write(
"\n## Dispatcher Timeline\n\n"
"> Dispatcher-owned. Workers and reviewers do not edit this section.\n\n"
"| seq | time | event | task | role | attempt | model | result | locator |\n"
"|---:|---|---|---|---|---:|---|---|---|\n"
)
sequence = 1 + max(
(
int(match.group(1))
for match in re.finditer(r"^\|\s*(\d+)\s*\|", text, re.MULTILINE)
),
default=0,
)
if not text.endswith("\n"):
stream.write("\n")
def cell(value: Any) -> str:
return str(value).replace("|", r"\|").replace("\n", " ")
stream.write(
f"| {sequence} | {work_log_now_iso()} | {cell(event)} | "
f"{cell(task.name)} | "
f"{cell(role)} | {attempt} | {cell(model)} | {cell(result)} | "
f"{cell(locator.resolve())} |\n"
)
stream.flush()
finally:
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
return path
def safe_name(value: str) -> str:
return re.sub(r"[^A-Za-z0-9_.-]+", "__", value).strip("_") or "task"
@dataclass(frozen=True)
class AgentSpec:
cli: str
model: str
display: str
local_pi: bool = False
@dataclass(frozen=True)
class PiSessionState:
phase: str
expected_tool_call_ids: tuple[str, ...] = ()
completed_tool_call_ids: tuple[str, ...] = ()
pending_tool_call_ids: tuple[str, ...] = ()
reason: str = ""
@dataclass
class Task:
name: str
directory: Path
plan: Path | None
review: Path | None
user_review: Path | None
recovery: bool
errors: list[str] = field(default_factory=list)
index: int = 0
deps: tuple[str, ...] = ()
write_set: set[str] = field(default_factory=set)
write_set_known: bool = False
plan_hash: str = "none"
lane: str | None = None
grade: int | None = None
def next_execution_identity(
store: StateStore,
task: Task,
role: str,
) -> tuple[int, str]:
attempt = store.next_attempt(task, role)
identity = (
f"{safe_name(task.name)}__p{plan_number(task)}__{role}__a{attempt:02d}"
)
return attempt, identity
class StateStore:
def __init__(self, workspace: Path):
workspace_id = hashlib.sha256(str(workspace).encode()).hexdigest()[:16]
git_marker = workspace / ".git"
git_directory: Path | None = None
if git_marker.is_dir():
git_directory = git_marker
elif git_marker.is_file():
marker = git_marker.read_text(encoding="utf-8", errors="replace").strip()
if marker.startswith("gitdir:"):
candidate = Path(marker.split(":", 1)[1].strip())
git_directory = candidate if candidate.is_absolute() else (workspace / candidate).resolve()
candidates = []
if git_directory is not None:
candidates.append(git_directory / "agent-task-dispatcher")
state_base = Path(os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local" / "state")))
candidates.append(state_base / "agent-task-dispatcher" / workspace_id)
self.root = candidates[-1]
last_error: OSError | None = None
for candidate in candidates:
try:
candidate.mkdir(parents=True, exist_ok=True)
self.root = candidate
last_error = None
break
except OSError as exc:
last_error = exc
if last_error is not None:
raise RuntimeError(f"dispatcher state 디렉터리를 만들 수 없다: {candidates}") from last_error
self.path = self.root / "state.json"
self.runs = self.root / "runs"
self.runs.mkdir(exist_ok=True)
self.lock_stream = (self.root / "dispatcher.lock").open("a+", encoding="utf-8")
try:
fcntl.flock(self.lock_stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
self.lock_stream.close()
raise RuntimeError(f"같은 workspace의 dispatcher가 이미 실행 중이다: {self.root}") from exc
if self.path.exists():
try:
self.data = json.loads(self.path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
self.lock_stream.close()
raise RuntimeError(
f"dispatcher state를 읽을 수 없다: {self.path}"
) from exc
if not isinstance(self.data, dict):
self.lock_stream.close()
raise RuntimeError(f"dispatcher state가 object가 아니다: {self.path}")
else:
self.data = {"tasks": {}, "attempt_counters": {}}
def save(self) -> None:
write_json(self.path, self.data)
def close(self) -> None:
if not self.lock_stream.closed:
self.lock_stream.close()
def task_state(self, task: Task) -> dict[str, Any]:
tasks = self.data.setdefault("tasks", {})
current = tasks.get(task.name)
if not current or current.get("plan_hash") != task.plan_hash:
current = {
"plan_hash": task.plan_hash,
"worker_done": False,
"worker_cli": None,
"worker_model": None,
"selfcheck_done": False,
"blocked": None,
"active_stage": None,
"active_locator": None,
"review_no_progress": 0,
"selfcheck_incomplete": 0,
"recovery_failures": {},
}
tasks[task.name] = current
self.save()
return current
def peek_task_state(self, task: Task) -> dict[str, Any]:
current = self.data.get("tasks", {}).get(task.name)
if current and current.get("plan_hash") == task.plan_hash:
return dict(current)
return {
"plan_hash": task.plan_hash,
"worker_done": False,
"worker_cli": None,
"worker_model": None,
"selfcheck_done": False,
"blocked": None,
"active_stage": None,
"active_locator": None,
"review_no_progress": 0,
"selfcheck_incomplete": 0,
"recovery_failures": {},
}
def update_task(self, task: Task, **values: Any) -> None:
state = self.task_state(task)
state.update(values)
self.save()
def mark_active(self, task: Task, stage: str, locator: Path | None = None) -> None:
self.update_task(
task,
active_stage=stage,
active_locator=str(locator) if locator else None,
active_started_at=now_iso(),
)
def clear_active(self, task: Task) -> None:
self.update_task(
task,
active_stage=None,
active_locator=None,
active_started_at=None,
)
def next_attempt(self, task: Task, role: str) -> int:
key = f"{task.name}|{task.plan_hash}|{role}"
counters = self.data.setdefault("attempt_counters", {})
number = int(counters.get(key, 0))
counters[key] = number + 1
self.save()
return number
def clear_blocked(self, task_group: str | None = None) -> None:
prefix = f"{task_group}/" if task_group else None
for task_name, value in self.data.get("tasks", {}).items():
if (
task_group is not None
and task_name != task_group
and not task_name.startswith(prefix)
):
continue
value["blocked"] = None
value["review_no_progress"] = 0
value["selfcheck_incomplete"] = 0
value["recovery_failures"] = {}
self.save()
def prepare_orchestration(
self,
scope: str,
tasks: list[Task],
workspace: Path,
) -> None:
orchestrations = self.data.setdefault("orchestrations", {})
current = orchestrations.get(scope)
if current is None or (current.get("status") == "complete" and tasks):
current = {"status": "running", "tasks": {}}
orchestrations[scope] = current
changed = False
tracked = current.setdefault("tasks", {})
for task in tasks:
record = tracked.get(task.name)
if record is None:
tracked[task.name] = {
"status": "active",
"archive": None,
"archive_baseline": [
str(path.resolve())
for path in matching_archive_directories_by_name(
workspace,
task.name,
require_complete=False,
)
],
}
changed = True
continue
if record.get("status") != "complete" and (
record.get("status") != "active" or "reason" in record
):
record["status"] = "active"
record.pop("reason", None)
changed = True
if changed or current.get("status") != "running":
current["status"] = "running"
self.save()
def mark_orchestration_task_complete(
self,
scope: str,
task_name: str,
archive: str | Path,
) -> None:
archive_path = Path(archive).resolve()
if not archive_path.is_dir() or not (archive_path / "complete.log").is_file():
raise RuntimeError(
f"완료 archive에 complete.log가 없다: task={task_name} archive={archive_path}"
)
current = self.data.setdefault("orchestrations", {}).setdefault(
scope, {"status": "running", "tasks": {}}
)
tracked = current.setdefault("tasks", {})
record = tracked.setdefault(
task_name,
{"status": "active", "archive": None, "archive_baseline": []},
)
record.update(status="complete", archive=str(archive_path))
record.pop("reason", None)
self.save()
cleanup_completed_task_attempt_logs(self.runs, task_name)
def mark_orchestration_blocked(
self,
scope: str,
outcomes: dict[str, tuple[str, str]],
) -> None:
current = self.data.setdefault("orchestrations", {}).setdefault(
scope, {"status": "running", "tasks": {}}
)
current["status"] = "blocked"
tracked = current.setdefault("tasks", {})
for task_name, (status, reason) in outcomes.items():
record = tracked.setdefault(
task_name,
{
"status": "active",
"archive": None,
"archive_baseline": [],
},
)
if record.get("status") == "complete":
continue
record.update(status=status, reason=reason)
self.save()
def reconcile_orchestration(
self,
scope: str,
workspace: Path,
active_or_running: set[str],
) -> tuple[dict[str, str], dict[str, str]]:
current = self.data.setdefault("orchestrations", {}).setdefault(
scope, {"status": "running", "tasks": {}}
)
completed: dict[str, str] = {}
errors: dict[str, str] = {}
changed = False
for task_name, record in current.setdefault("tasks", {}).items():
if record.get("status") == "complete":
archive = str(record.get("archive") or "")
if archive and (Path(archive) / "complete.log").is_file():
completed[task_name] = archive
else:
errors[task_name] = "persisted complete archive가 유효하지 않다"
continue
if task_name in active_or_running:
continue
baseline = set(str(path) for path in record.get("archive_baseline", []))
candidates = [
path
for path in matching_archive_directories_by_name(workspace, task_name)
if str(path.resolve()) not in baseline
]
if len(candidates) == 1:
archive = str(candidates[0].resolve())
record.update(status="complete", archive=archive)
completed[task_name] = archive
changed = True
elif not candidates:
errors[task_name] = (
"관찰된 task가 active와 새 complete.log archive 모두에서 사라졌다"
)
else:
errors[task_name] = (
"새 complete.log archive가 여러 개라 완료 경로를 확정할 수 없다: "
+ ",".join(str(path) for path in candidates)
)
if changed:
self.save()
for task_name in completed:
cleanup_completed_task_attempt_logs(self.runs, task_name)
return completed, errors
def orchestration_tasks(self, scope: str) -> set[str]:
current = self.data.get("orchestrations", {}).get(scope, {})
return set(current.get("tasks", {}))
def mark_orchestration_complete(self, scope: str) -> None:
current = self.data.setdefault("orchestrations", {}).setdefault(
scope, {"status": "running", "tasks": {}}
)
current["status"] = "complete"
self.save()
def parse_route(plan: Path | None) -> tuple[str | None, int | None]:
if plan is None:
return None, None
match = PLAN_RE.match(plan.name)
if not match:
return None, None
return match.group(1), int(match.group(2))
def parse_task_name(task_root: Path, directory: Path) -> str:
return directory.relative_to(task_root).as_posix()
def extract_write_set(plan: Path | None, workspace: Path) -> tuple[set[str], bool]:
if plan is None or not plan.exists():
return set(), False
text = plan.read_text(encoding="utf-8", errors="replace")
match = re.search(r"^## 수정 파일 요약\s*$([\s\S]*?)(?=^##\s|\Z)", text, re.MULTILINE)
if not match:
return set(), False
result: set[str] = set()
invalid = False
for line in match.group(1).splitlines():
if not line.lstrip().startswith("|"):
continue
cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
if not cells:
continue
for value in re.findall(r"`([^`]+)`", cells[0]):
normalized = re.sub(r":\d+(?::\d+)?$", "", value.strip())
if normalized and not normalized.startswith(("http://", "https://")):
if any(character in normalized for character in "*?[]"):
invalid = True
continue
candidate = Path(normalized)
resolved = (
candidate.resolve()
if candidate.is_absolute()
else (workspace / candidate).resolve()
)
try:
resolved.relative_to(workspace)
except ValueError:
invalid = True
continue
if resolved == workspace or resolved.is_dir():
invalid = True
continue
result.add(str(resolved))
return result, bool(result) and not invalid
def latest_verdict_log(directory: Path) -> Path | None:
logs = sorted(directory.glob("code_review_*.log"), key=lambda p: p.stat().st_mtime)
if not logs:
return None
newest = logs[-1]
return newest if read_verdict(newest) else None
def matching_plan_log(directory: Path, review_log: Path | None) -> Path | None:
if review_log is None:
return None
review_identity = plan_identity(review_log)
matches = [
path
for path in directory.glob("plan_*.log")
if plan_identity(path) == review_identity
]
return max(matches, key=lambda path: path.stat().st_mtime_ns) if matches else None
def read_task_directory(workspace: Path, directory: Path) -> Task | None:
"""Read one already-known task directory without scanning the task group."""
task_root = workspace / "agent-task"
if not directory.is_dir():
return None
plans = sorted(p for p in directory.iterdir() if p.is_file() and PLAN_RE.match(p.name))
reviews = sorted(p for p in directory.iterdir() if p.is_file() and REVIEW_RE.match(p.name))
users = sorted(directory.glob("USER_REVIEW.md"))
complete = directory / "complete.log"
recovery_log = latest_verdict_log(directory)
if not plans and not reviews and not users and not complete.exists() and recovery_log is None:
return None
name = parse_task_name(task_root, directory)
errors: list[str] = []
if len(plans) > 1:
errors.append(f"active PLAN이 {len(plans)}개다")
if len(reviews) > 1:
errors.append(f"active CODE_REVIEW가 {len(reviews)}개다")
if len(users) > 1:
errors.append(f"USER_REVIEW가 {len(users)}개다")
if users and (plans or reviews):
errors.append("USER_REVIEW stop state와 active PLAN/CODE_REVIEW가 공존한다")
plan = plans[0] if len(plans) == 1 else None
review = reviews[0] if len(reviews) == 1 else None
recovery = complete.exists() or recovery_log is not None
if bool(plan) != bool(review) and not recovery:
errors.append("active PLAN/CODE_REVIEW pair가 불완전하다")
relative = directory.relative_to(task_root)
subtask = relative.parts[1] if len(relative.parts) == 2 else None
index = 0
deps: tuple[str, ...] = ()
if subtask:
match = SUBTASK_RE.match(subtask)
if match:
index = int(match.group("index"))
deps = tuple((match.group("deps") or "").split(",")) if match.group("deps") else ()
else:
errors.append(f"split subtask 이름이 계약과 다르다: {subtask}")
lane, grade = parse_route(plan)
recovery_plan = matching_plan_log(directory, recovery_log)
write_set_source = plan or recovery_plan
write_set, write_set_known = extract_write_set(write_set_source, workspace)
if plan is not None:
metadata = PLAN_IDENTITY_RE.search(
plan.read_text(encoding="utf-8", errors="replace")[:1024]
)
if metadata is None:
errors.append("PLAN task/plan/tag metadata를 판별할 수 없다")
elif metadata.group("task") != name:
errors.append(
f"PLAN task metadata가 디렉터리와 다르다: {metadata.group('task')}"
)
return Task(
name=name,
directory=directory,
plan=plan,
review=review,
user_review=users[0] if len(users) == 1 else None,
recovery=recovery,
errors=errors,
index=index,
deps=deps,
write_set=write_set,
write_set_known=write_set_known,
plan_hash=(
plan_identity(plan)
if plan
else sha256_file(
recovery_log
or (users[0] if len(users) == 1 else complete)
)
),
lane=lane,
grade=grade,
)
def scan_tasks(workspace: Path, task_group: str | None) -> list[Task]:
task_root = workspace / "agent-task"
if not task_root.is_dir():
raise RuntimeError(f"agent-task 디렉터리가 없다: {task_root}")
directories: list[Path] = []
groups = [task_root / task_group] if task_group else sorted(
p for p in task_root.iterdir() if p.is_dir() and p.name != "archive"
)
for group in groups:
if not group.is_dir():
continue
directories.append(group)
directories.extend(sorted(p for p in group.iterdir() if p.is_dir()))
tasks = [
task
for directory in directories
if (task := read_task_directory(workspace, directory)) is not None
]
return sorted(tasks, key=lambda task: (task.index, task.name))
def dependency_candidates(workspace: Path, task: Task, predecessor: str) -> list[Path]:
parts = task.name.split("/")
if len(parts) != 2:
return []
group = parts[0]
task_root = workspace / "agent-task"
found: list[Path] = []
active_group = task_root / group
for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"):
found.extend(active_group.glob(pattern))
archive = task_root / "archive"
if archive.is_dir():
for year in archive.iterdir():
if not year.is_dir():
continue
for month in year.iterdir():
archived_group = month / group
if not archived_group.is_dir():
continue
for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"):
found.extend(archived_group.glob(pattern))
return sorted(set(path.resolve() for path in found))
def dependency_state(workspace: Path, task: Task) -> tuple[bool, str]:
missing: list[str] = []
ambiguous: list[str] = []
for predecessor in task.deps:
candidates = dependency_candidates(workspace, task, predecessor)
if not candidates:
missing.append(predecessor)
elif len(candidates) > 1:
ambiguous.append(f"{predecessor}={','.join(str(p) for p in candidates)}")
if ambiguous:
return False, "dependency ambiguity: " + "; ".join(ambiguous)
if missing:
return False, "predecessor complete.log 대기: " + ",".join(missing)
return True, "ready"
def route_agent(task: Task) -> AgentSpec:
if task.lane is None or task.grade is None:
raise RuntimeError("PLAN route를 판별할 수 없다")
grade = task.grade
if task.lane == "local":
if grade <= 6:
return AgentSpec("pi", "ornith-fast", "pi/iop/ornith-fast", local_pi=True)
if grade <= 8:
return AgentSpec("pi", "laguna-s:2.1", "pi/iop/laguna-s:2.1", local_pi=True)
return AgentSpec("claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh")
if grade <= 2:
return AgentSpec("agy", "Gemini 3.5 Flash (Low)", "agy/Gemini 3.5 Flash (Low)")
if grade <= 4:
return AgentSpec("agy", "Gemini 3.5 Flash (Medium)", "agy/Gemini 3.5 Flash (Medium)")
if grade <= 6:
return AgentSpec("agy", "Gemini 3.5 Flash (High)", "agy/Gemini 3.5 Flash (High)")
if grade <= 8:
return AgentSpec("claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh")
return AgentSpec("codex", "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh")
def plan_number(task: Task) -> int:
if task.plan and task.plan.exists():
match = PLAN_IDENTITY_RE.search(
task.plan.read_text(encoding="utf-8", errors="replace")[:1024]
)
if match:
return int(match.group("plan"))
return 0
def task_requires_selfcheck(task: Task) -> bool:
return task.lane == "local" and task.grade is not None and task.grade <= 8
def concrete_user_review_value(value: str) -> bool:
normalized = value.strip().strip("`").strip()
normalized = re.sub(r"^-\s*", "", normalized).strip()
if not normalized or re.search(r"\{[^}]+\}|<[^>]+>", normalized):
return False
return normalized.casefold() not in {
"-",
"n/a",
"na",
"none",
"unknown",
"미정",
"없음",
"해당 없음",
}
def user_review_blocker_state(path: Path) -> tuple[bool, str]:
if not path.is_file():
return False, "파일이 없다"
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
return False, f"파일을 읽을 수 없다: {exc}"
status = markdown_section(text, "상태").strip().strip("`")
if status != "USER_REVIEW":
return False, "상태가 USER_REVIEW가 아니다"
reason = markdown_section(text, "사유")
if not re.search(r"(?m)^-\s*유형:\s*milestone-lock\s*$", reason):
return False, "milestone-lock 유형이 아니다"
target = re.search(r"(?m)^-\s*연결 대상:\s*(.+?)\s*$", reason)
target_value = target.group(1) if target else ""
if (
not concrete_user_review_value(target_value)
or "agent-roadmap/" not in target_value
or "/milestones/" not in target_value
or ".md" not in target_value
):
return False, "구체적인 Milestone 연결 대상이 없다"
evidence = markdown_section(text, "차단 근거")
evidence_line = re.search(r"(?m)^-\s*차단 판단 근거:\s*(.+?)\s*$", evidence)
if evidence_line is None or not concrete_user_review_value(
evidence_line.group(1)
):
return False, "구체적인 차단 판단 근거가 없다"
decision = markdown_section(text, "연결 결정 필요")
unresolved = [
value
for value in re.findall(r"(?m)^-\s*\[\s\]\s+(.+?)\s*$", decision)
if concrete_user_review_value(value)
]
if not unresolved:
return False, "미해결 연결 결정 항목이 없다"
resume = markdown_section(text, "재개 조건")
resume_conditions = [
line
for line in resume.splitlines()
if concrete_user_review_value(line)
]
if not resume_conditions:
return False, "구체적인 재개 조건이 없다"
return True, "unresolved milestone-lock decision"
def task_stage(task: Task, state: dict[str, Any]) -> str:
if task.errors:
return "blocked"
if task.user_review:
if task.plan is not None or task.review is not None:
return "blocked"
blocking, _ = user_review_blocker_state(task.user_review)
return "user-review" if blocking else "blocked"
if task.recovery and (task.plan is None or task.review is None):
return "review"
if task.review and task.review.exists():
text = task.review.read_text(encoding="utf-8", errors="replace")
if verdict_from_text(text):
return "review"
if state.get("worker_done"):
if task_requires_selfcheck(task) and not state.get("selfcheck_done"):
return "selfcheck"
return "review"
if task.review and task.review.exists() and FINAL_CHECK_RE.search(text):
return "review"
return "worker"
def markdown_section(text: str, heading: str) -> str:
match = re.search(rf"^## {re.escape(heading)}[ \t]*$", text, re.MULTILINE)
if match is None:
return ""
next_heading = re.search(r"^##\s+", text[match.end():], re.MULTILINE)
end = match.end() + next_heading.start() if next_heading else len(text)
return text[match.end():end].strip()
def implementation_review_errors(task: Task) -> list[str]:
if task.review is None or not task.review.is_file():
return ["CODE_REVIEW 파일 없음"]
text = task.review.read_text(encoding="utf-8", errors="replace")
errors: list[str] = []
checklist = markdown_section(text, "구현 체크리스트")
if not checklist or re.search(r"^-\s+\[\s\]", checklist, re.MULTILINE):
errors.append("구현 체크리스트 미완료")
completion = markdown_section(text, "구현 항목별 완료 여부")
if not completion or re.search(r"\|\s*\[\s\]\s*\|", completion):
errors.append("구현 항목 완료 여부 미작성")
for heading in ("계획 대비 변경 사항", "주요 설계 결정"):
body = markdown_section(text, heading)
if not body or re.fullmatch(r"_[^_]*_", body, re.DOTALL):
errors.append(f"{heading} 미작성")
verification = markdown_section(text, "검증 결과")
if not verification or re.search(r"_미실행(?:[^_]*)_", verification):
errors.append("검증 결과 미작성")
if not FINAL_CHECK_RE.search(text):
errors.append("CODE_REVIEW 동기화 체크 미완료")
return errors
def classify_failure_with_evidence(output: str) -> tuple[str, str | None]:
lines = output.splitlines()
for category, patterns in PROMOTABLE_PATTERNS.items():
for line in reversed(lines):
lowered = line.lower()
if any(re.search(pattern, lowered, re.DOTALL) for pattern in patterns):
return category, line
return "generic-error", None
def classify_failure(output: str) -> str:
return classify_failure_with_evidence(output)[0]
def termination_signal(return_code: int) -> tuple[str, bool] | None:
signal_number: int | None = None
inferred = False
if return_code < 0:
signal_number = -return_code
elif return_code > 128:
signal_number = return_code - 128
inferred = True
if signal_number is None:
return None
try:
return signal.Signals(signal_number).name, inferred
except ValueError:
return None
def failure_report_lines(failure: str, locator: Path) -> list[str]:
record: dict[str, Any] = {}
try:
record = json.loads(locator.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
pass
failure_class = str(record.get("failure_class") or failure)
source = str(record.get("failure_source") or "unverified")
provider_confirmed = bool(
record.get("provider_transport_failure_confirmed", False)
)
lines = [
f"failure_class={failure_class}",
f"failure_source={source}",
"provider_transport_failure_confirmed="
f"{str(provider_confirmed).lower()}",
]
if record.get("dispatcher_pid") is not None:
lines.append(f"dispatcher_pid={record['dispatcher_pid']}")
if record.get("dispatcher_source_sha256"):
lines.append(
f"dispatcher_source_sha256={record['dispatcher_source_sha256']}"
)
source_matches_loaded = record.get("dispatcher_source_matches_loaded")
if source_matches_loaded is not None:
lines.append(
"dispatcher_source_matches_loaded="
f"{str(bool(source_matches_loaded)).lower()}"
)
if (
source_matches_loaded is False
and record.get("dispatcher_source_current_sha256")
):
lines.append(
"dispatcher_source_current_sha256="
f"{record['dispatcher_source_current_sha256']}"
)
if provider_confirmed:
evidence_source = record.get("failure_evidence_source")
evidence = record.get("failure_evidence_excerpt")
if evidence_source:
lines.append(f"provider_evidence_source={evidence_source}")
if evidence:
rendered = str(evidence).replace("\r", r"\r").replace("\n", r"\n")
lines.append(f"provider_evidence={rendered}")
if failure_class == "session-stall":
lines.extend(
[
f"timeout_phase={record.get('pi_session_phase') or 'unknown'}",
f"timeout_seconds={record.get('session_stall_seconds') or 'unknown'}",
"termination_initiator="
f"{record.get('termination_initiator') or 'dispatcher'}",
]
)
elif failure_class == "process-terminated":
lines.extend(
[
f"termination_signal={record.get('termination_signal') or 'unknown'}",
"termination_initiator="
f"{record.get('termination_initiator') or 'unknown'}",
]
)
lines.append(f"locator={locator}")
return lines
def terminal_diagnostic(cli: str, channel: str, line: str) -> str | None:
if channel == "stderr":
return line
try:
value = json.loads(line)
except json.JSONDecodeError:
if cli == "agy" and re.match(
r"^\s*(?:error|fatal|provider error|model error)\b", line, re.IGNORECASE
):
return line
return None
event_type = str(value.get("type", ""))
if cli == "codex" and event_type in {"turn.failed", "error"}:
return json.dumps(value.get("error", value), ensure_ascii=False)
if cli == "claude":
subtype = str(value.get("subtype", ""))
if event_type == "result" and (value.get("is_error") or subtype.startswith("error")):
return str(value.get("result") or value)
if event_type == "system" and subtype.startswith("error"):
return json.dumps(value, ensure_ascii=False)
return None
def codex_collaboration_tool(line: str) -> str | None:
try:
value = json.loads(line)
except json.JSONDecodeError:
return None
item = value.get("item") or {}
if (
value.get("type") == "item.started"
and item.get("type") == "collab_tool_call"
and item.get("tool")
):
return str(item["tool"])
return None
async def terminate_process_group(
process: asyncio.subprocess.Process,
grace_seconds: float = 5,
) -> None:
"""Terminate the exact subprocess group and escalate if descendants remain."""
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
if process.returncode is None:
await process.wait()
return
if process.returncode is None:
try:
await asyncio.wait_for(process.wait(), timeout=grace_seconds)
except TimeoutError:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
await process.wait()
return
try:
os.killpg(process.pid, 0)
except ProcessLookupError:
return
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
def agy_log_diagnostics(path: Path) -> list[str]:
if not path.exists():
return []
diagnostics: list[str] = []
for line in path.read_text(encoding="utf-8", errors="replace").splitlines()[-200:]:
if re.search(r"\b(?:error|fatal)\b", line, re.IGNORECASE):
diagnostics.append(line)
elif re.search(r"RESOURCE_EXHAUSTED|rate.?limit|model.{0,40}unavailable", line, re.IGNORECASE):
diagnostics.append(line)
return diagnostics
def promoted_spec(spec: AgentSpec, recovery_count: int) -> AgentSpec | None:
if spec.cli == "agy":
return AgentSpec("claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh")
if spec.cli == "claude":
return AgentSpec("codex", "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh")
if spec.cli == "codex" and recovery_count < 1:
return AgentSpec("codex", "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh")
return None
def render_json_line(cli: str, line: str) -> tuple[list[str], str | None]:
try:
value = json.loads(line)
except json.JSONDecodeError:
return [line.rstrip()], None
session_id = value.get("thread_id") or value.get("session_id")
rendered: list[str] = []
if cli == "codex":
if value.get("type") == "thread.started" and session_id:
rendered.append(f"session={session_id}")
item = value.get("item") or {}
item_type = item.get("type")
if item_type == "agent_message" and item.get("text"):
rendered.extend(str(item["text"]).splitlines())
elif item_type == "command_execution":
rendered.append(f"$ {item.get('command', '')} (exit={item.get('exit_code', '?')})")
elif item_type in {"mcp_tool_call", "web_search"}:
rendered.append(f"{item_type}: {item.get('server', '')} {item.get('tool', item.get('query', ''))}")
elif value.get("type") == "turn.failed":
rendered.append(str(value.get("error", value)))
elif cli == "claude":
message = value.get("message") or {}
for block in message.get("content") or []:
if block.get("type") == "text":
rendered.extend(str(block.get("text", "")).splitlines())
elif block.get("type") == "tool_use":
rendered.append(f"tool={block.get('name', '')}")
if value.get("type") == "result" and value.get("result"):
rendered.extend(str(value["result"]).splitlines())
session_id = session_id or value.get("session_id")
return rendered, str(session_id) if session_id else None
def native_session_path(cli: str, workspace: Path, session_id: str | None, attempt_dir: Path) -> str | None:
if cli == "claude" and session_id:
encoded = str(workspace).replace("/", "-")
return str(Path.home() / ".claude" / "projects" / encoded / f"{session_id}.jsonl")
if cli == "pi" and session_id:
matches = list((attempt_dir / "pi-sessions").glob(f"*{session_id}*.jsonl"))
return str(matches[0]) if matches else str(attempt_dir / "pi-sessions")
if cli == "codex" and session_id:
matches = list((Path.home() / ".codex" / "sessions").glob(f"**/*{session_id}*.jsonl"))
return str(matches[0]) if matches else str(Path.home() / ".codex" / "sessions")
return None
def native_session_mtime_ns(path: str | None) -> int | None:
if not path:
return None
candidate = Path(path)
return candidate.stat().st_mtime_ns if candidate.is_file() else None
def reverse_jsonl_lines(path: Path):
with path.open("rb") as stream:
stream.seek(0, os.SEEK_END)
position = stream.tell()
buffer = b""
while position > 0:
read_size = min(8192, position)
position -= read_size
stream.seek(position)
buffer = stream.read(read_size) + buffer
lines = buffer.split(b"\n")
buffer = lines[0]
for line in reversed(lines[1:]):
if line.strip():
yield line
if buffer.strip():
yield buffer
def pi_session_header_version(path: Path) -> int | None:
with path.open("rb") as stream:
first_line = stream.readline()
if not first_line.strip():
return None
header = json.loads(first_line)
if not isinstance(header, dict) or header.get("type") != "session":
return None
version = header.get("version")
return version if isinstance(version, int) else None
def pi_native_session_state(path: str | None) -> PiSessionState:
if not path:
return PiSessionState("starting", reason="native-session-path-missing")
candidate = Path(path)
if not candidate.is_file():
return PiSessionState("starting", reason="native-session-file-missing")
completed_ids: list[str] = []
expected_entry_id: str | None = None
active_leaf_found = False
try:
version = pi_session_header_version(candidate)
if version != PI_SESSION_SCHEMA_VERSION:
return PiSessionState(
"unknown",
reason=(
f"unsupported-session-version:{version}"
if version is not None
else "session-header-invalid"
),
)
for raw_line in reverse_jsonl_lines(candidate):
value = json.loads(raw_line)
if not isinstance(value, dict):
return PiSessionState("unknown", reason="invalid-entry-schema")
if value.get("type") == "session":
break
entry_id = value.get("id")
parent_id = value.get("parentId")
if (
not isinstance(entry_id, str)
or not entry_id
or "parentId" not in value
or (parent_id is not None and not isinstance(parent_id, str))
):
return PiSessionState("unknown", reason="invalid-entry-identity")
if active_leaf_found and entry_id != expected_entry_id:
continue
active_leaf_found = True
expected_entry_id = parent_id
if value.get("type") != "message":
continue
message = value.get("message")
if not isinstance(message, dict):
return PiSessionState("unknown", reason="invalid-message-schema")
role = message.get("role")
if role == "toolResult":
tool_call_id = message.get("toolCallId")
if not isinstance(tool_call_id, str) or not tool_call_id:
return PiSessionState(
"unknown", reason="tool-result-id-missing"
)
if tool_call_id in completed_ids:
return PiSessionState(
"unknown", reason="duplicate-tool-result-id"
)
completed_ids.append(tool_call_id)
continue
if role == "user":
if completed_ids:
return PiSessionState(
"unknown", reason="tool-results-without-assistant"
)
return PiSessionState("awaiting-model", reason="user-message")
if role != "assistant":
return PiSessionState(
"unknown", reason=f"unsupported-message-role:{role}"
)
content = message.get("content")
if not isinstance(content, list):
return PiSessionState(
"unknown", reason="assistant-content-not-list"
)
if any(
not isinstance(block, dict)
or block.get("type") not in {"text", "thinking", "toolCall"}
for block in content
):
return PiSessionState(
"unknown", reason="unsupported-assistant-content"
)
tool_calls = [
block
for block in content
if isinstance(block, dict) and block.get("type") == "toolCall"
]
if not tool_calls:
if completed_ids:
return PiSessionState(
"unknown", reason="tool-results-without-tool-calls"
)
return PiSessionState("finishing", reason="assistant-final")
expected_ids: list[str] = []
for tool_call in tool_calls:
tool_call_id = tool_call.get("id")
if not isinstance(tool_call_id, str) or not tool_call_id:
return PiSessionState(
"unknown", reason="tool-call-id-missing"
)
if tool_call_id in expected_ids:
return PiSessionState(
"unknown", reason="duplicate-tool-call-id"
)
expected_ids.append(tool_call_id)
unexpected_ids = [
tool_call_id
for tool_call_id in completed_ids
if tool_call_id not in expected_ids
]
if unexpected_ids:
return PiSessionState(
"unknown", reason="tool-result-id-not-in-latest-batch"
)
completed_set = set(completed_ids)
completed = tuple(
tool_call_id
for tool_call_id in expected_ids
if tool_call_id in completed_set
)
pending = tuple(
tool_call_id
for tool_call_id in expected_ids
if tool_call_id not in completed_set
)
return PiSessionState(
"tool-running" if pending else "awaiting-model",
expected_tool_call_ids=tuple(expected_ids),
completed_tool_call_ids=completed,
pending_tool_call_ids=pending,
reason=(
"pending-tool-results"
if pending
else "all-tool-results-recorded"
),
)
if completed_ids:
return PiSessionState(
"unknown", reason="tool-results-without-assistant"
)
if active_leaf_found and expected_entry_id is not None:
return PiSessionState("unknown", reason="active-branch-parent-missing")
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
return PiSessionState("unknown", reason="unreadable-jsonl")
return PiSessionState("starting", reason="no-message-events")
def pi_native_session_phase(path: str | None) -> str:
return pi_native_session_state(path).phase
def pi_stall_timeout_seconds(phase: str) -> int:
if phase == "awaiting-model":
return PI_MODEL_RESPONSE_STALL_SECONDS
if phase in {"tool-running", "unknown"}:
return PI_TOOL_STALL_SECONDS
return PI_SESSION_START_STALL_SECONDS
def log_tail_excerpt(path: Path, *, byte_limit: int = 8192, char_limit: int = 2000) -> str:
"""Return a bounded recent log excerpt without loading a long reasoning stream."""
try:
with path.open("rb") as stream:
stream.seek(max(0, path.stat().st_size - byte_limit))
text = stream.read().decode("utf-8", errors="replace")
except OSError as exc:
return f"<stream log unavailable: {exc}>"
return text[-char_limit:]
def process_is_alive(value: Any) -> bool:
"""Return whether an attempt/dispatcher PID still exists without signalling it."""
try:
pid = int(value)
if pid <= 0:
return False
os.kill(pid, 0)
except (TypeError, ValueError, OSError):
return False
return True
def external_active_is_live(state: dict[str, Any]) -> tuple[bool, str]:
raw_locator = state.get("active_locator")
if not raw_locator:
return False, "active locator 없음"
target = Path(str(raw_locator))
locator_path = target if target.name == "locator.json" else target / "locator.json"
locator: dict[str, Any] = {}
if locator_path.is_file():
try:
locator = json.loads(locator_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return False, f"locator 판독 실패: {locator_path}"
status = str(locator.get("status") or "")
if status and status != "running":
return False, f"locator status={status}"
# The stream may legitimately remain quiet during long reasoning. A live
# process is stronger evidence than a locator or dispatcher heartbeat, and
# prevents a second dispatcher from duplicating an active attempt.
for field in ("agent_pid", "dispatcher_pid"):
if process_is_alive(locator.get(field)):
return True, f"{field}={locator[field]} alive; output stream is monitored"
native_raw = locator.get("native_session_path")
native = Path(str(native_raw)) if native_raw else None
if native and native.is_dir():
sessions = list(native.glob("*.jsonl"))
native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
if native is None or not native.is_file():
roots = [target] if target.is_dir() else [target.parent]
sessions = [
path
for root in roots
for path in (*root.glob("*.jsonl"), *root.glob("pi-sessions/*.jsonl"))
]
native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
now = datetime.now(timezone.utc).timestamp()
cli = str(locator.get("cli") or "")
stream_progress_at: float | None = None
stream_raw = locator.get("stream_log")
stream = Path(str(stream_raw)) if stream_raw else None
if stream and stream.is_file():
stream_progress_at = stream.stat().st_mtime
if native and native.is_file():
native_progress_at = native.stat().st_mtime
progress_at = max(native_progress_at, stream_progress_at or 0.0)
inactive = max(0.0, now - progress_at)
if cli == "pi":
phase = pi_native_session_phase(str(native))
# Only an exact incomplete toolCall -> toolResult batch is a tool
# execution interval. Unknown/starting/model-reasoning states
# must never be treated as a stalled tool merely because their
# native event file is quiet.
timeout = (
PI_TOOL_STALL_SECONDS
if phase == "tool-running"
else PI_MODEL_RESPONSE_STALL_SECONDS
)
return (
inactive < timeout,
f"phase={phase} native+stream inactive={inactive:.1f}s timeout={timeout}s",
)
return (
inactive < CODEX_STREAM_STALL_SECONDS,
"native+stream inactive="
f"{inactive:.1f}s timeout={CODEX_STREAM_STALL_SECONDS}s",
)
if stream_progress_at is not None:
inactive = max(0.0, now - stream_progress_at)
return (
inactive < CODEX_STREAM_STALL_SECONDS,
f"stream inactive={inactive:.1f}s timeout={CODEX_STREAM_STALL_SECONDS}s",
)
return False, f"active 증거 없음: {raw_locator}"
def laguna_resume_locator(state: dict[str, Any]) -> Path | None:
raw_locator = state.get("active_locator")
if not raw_locator:
return None
locator = Path(str(raw_locator))
try:
record = json.loads(locator.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return None
if (
record.get("cli") != "pi"
or not str(record.get("model", "")).startswith("laguna-s")
or record.get("failure_class") not in {"context-limit", "session-stall"}
or record.get("status") != "failed"
):
return None
native_raw = record.get("native_session_path")
native = Path(str(native_raw)) if native_raw else None
if native is None or not native.exists():
return None
return locator
def agy_conversations() -> dict[Path, int]:
root = Path.home() / ".gemini" / "antigravity-cli" / "conversations"
if not root.is_dir():
return {}
return {path: path.stat().st_mtime_ns for path in root.glob("*.db")}
def build_command(
spec: AgentSpec,
prompt: str,
workspace: Path,
session_id: str,
attempt_dir: Path,
pi_resume_session: Path | None = None,
) -> list[str]:
if spec.cli == "codex":
return [
"codex", "exec", "--json", "-C", str(workspace), "-m", spec.model,
"-c", 'model_reasoning_effort="xhigh"',
"--dangerously-bypass-approvals-and-sandbox", prompt,
]
if spec.cli == "claude":
return [
"claude", "-p", "--output-format", "stream-json", "--verbose",
"--session-id", session_id, "--model", spec.model, "--effort", "xhigh",
"--dangerously-skip-permissions", prompt,
]
if spec.cli == "agy":
return [
"agy", "--print", "--print-timeout", "8h", "--model", spec.model,
"--dangerously-skip-permissions", "--log-file", str(attempt_dir / "agy-cli.log"), prompt,
]
if spec.cli == "pi":
command = [
"pi", "-p", "--mode", "json", "--approve", "--provider", "iop", "--model", spec.model,
"--thinking", "high",
]
if pi_resume_session is not None:
command.extend(
[
"--session", str(pi_resume_session),
"--session-dir", str(pi_resume_session.parent),
]
)
else:
command.extend(
[
"--session-id", session_id,
"--session-dir", str(attempt_dir / "pi-sessions"),
]
)
command.append(prompt)
return command
raise RuntimeError(f"지원하지 않는 CLI: {spec.cli}")
async def invoke(
workspace: Path,
store: StateStore,
task: Task,
role: str,
spec: AgentSpec,
prompt: str,
resume_locator: Path | None = None,
) -> tuple[int, str | None, Path]:
attempt, identity = next_execution_identity(store, task, role)
attempt_dir = store.runs / f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}__{identity}"
attempt_dir.mkdir(parents=True, exist_ok=False)
locator_path = attempt_dir / "locator.json"
stream_path = attempt_dir / "stream.log"
heartbeat_path = attempt_dir / "heartbeat.log"
session_id = str(uuid.uuid4())
pi_resume_session: Path | None = None
if spec.local_pi and resume_locator and resume_locator.is_file():
try:
prior = json.loads(resume_locator.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
prior = {}
prior_native = prior.get("native_session_path")
candidate = Path(str(prior_native)) if prior_native else None
if candidate and candidate.is_dir():
sessions = list(candidate.glob("*.jsonl"))
candidate = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
if candidate and candidate.is_file():
pi_resume_session = candidate
session_id = str(prior.get("session_id") or candidate.stem)
started_at = now_iso()
work_log_path = milestone_work_log_path(task)
record: dict[str, Any] = {
"execution_id": identity,
"task": task.name,
"plan_number": plan_number(task),
"role": role,
"attempt": attempt,
"workspace": str(workspace),
**dispatcher_source_provenance(),
"cli": spec.cli,
"model": spec.model,
"plan_path": str(task.plan) if task.plan else None,
"review_path": str(task.review) if task.review else None,
"session_id": session_id if spec.cli in {"claude", "pi"} else None,
"native_session_path": (
str(pi_resume_session)
if pi_resume_session is not None
else native_session_path(spec.cli, workspace, session_id, attempt_dir)
),
"output_log": str(stream_path),
"stream_log": str(stream_path),
"heartbeat_log": str(heartbeat_path),
"cli_log": str(attempt_dir / "agy-cli.log") if spec.cli == "agy" else None,
"work_log": str(work_log_path.resolve()),
"started_at": started_at,
"status": "running",
"resumed_from_locator": str(resume_locator) if pi_resume_session else None,
}
write_json(locator_path, record)
store.update_task(task, active_locator=str(locator_path))
prefix = f"[{task.directory.name}][{role}][a{attempt:02d}]"
print(f"{prefix} locator={locator_path}", flush=True)
try:
append_milestone_event(
task,
event="START",
execution_id=identity,
role=role,
attempt=attempt,
model=spec.display,
result="running",
locator=locator_path,
)
except OSError as exc:
line = f"milestone work log setup failed: {exc}"
heartbeat_path.write_text(line + "\n", encoding="utf-8")
record.update(
status="failed",
finished_at=now_iso(),
exit_code=1,
failure_class="work-log-setup",
failure_source="work-log",
provider_transport_failure_confirmed=False,
work_log_error=str(exc),
)
write_json(locator_path, record)
print(f"{prefix} {line}", flush=True)
return 1, "work-log-setup", locator_path
command = build_command(
spec,
prompt,
workspace,
session_id,
attempt_dir,
pi_resume_session=pi_resume_session,
)
before_agy = agy_conversations() if spec.cli == "agy" else {}
diagnostics: list[str] = []
diagnostic_origins: list[str] = []
control_violation: str | None = None
session_stall = False
try:
process = await asyncio.create_subprocess_exec(
*command,
cwd=workspace,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
limit=10 * 1024 * 1024,
start_new_session=True,
)
# Keep the child PID in the locator before monitoring output. If this
# dispatcher is interrupted, a later dispatcher can distinguish a
# genuinely live, silent model from a stale locator and must not launch
# a duplicate continuation.
record["agent_pid"] = process.pid
write_json(locator_path, record)
except FileNotFoundError:
line = f"command not found: {command[0]}"
heartbeat_path.write_text(line + "\n", encoding="utf-8")
failure_class = "generic-error"
try:
append_milestone_event(
task,
event="FINISH",
execution_id=identity,
role=role,
attempt=attempt,
model=spec.display,
result=f"failed:{failure_class}:127",
locator=locator_path,
)
except OSError as exc:
record["work_log_runtime_error"] = str(exc)
failure_class = "work-log-runtime-write"
record.update(
status="failed",
finished_at=now_iso(),
exit_code=127,
failure_class=failure_class,
failure_source=(
"work-log" if failure_class == "work-log-runtime-write" else "cli-launch"
),
provider_transport_failure_confirmed=False,
)
write_json(locator_path, record)
print(f"{prefix} {line}", flush=True)
return 127, failure_class, locator_path
readers: list[asyncio.Task[None]] = []
try:
assert process.stdout is not None and process.stderr is not None
queue: asyncio.Queue[tuple[str, bytes | None]] = asyncio.Queue()
async def pump(channel: str, stream: asyncio.StreamReader) -> None:
try:
while True:
raw = await stream.readline()
if not raw:
break
await queue.put((channel, raw))
finally:
await queue.put((channel, None))
readers = [
asyncio.create_task(pump("stdout", process.stdout)),
asyncio.create_task(pump("stderr", process.stderr)),
]
finished_streams = 0
loop = asyncio.get_running_loop()
last_native_mtime: int | None = None
last_stream_mtime: int | None = None
last_native_progress_at = loop.time()
last_stream_progress_at = loop.time()
with (
stream_path.open("w", encoding="utf-8") as stream_log,
heartbeat_path.open("a", encoding="utf-8") as heartbeat_log,
):
while finished_streams < len(readers):
try:
channel, raw = await asyncio.wait_for(
queue.get(), timeout=STREAM_HEARTBEAT_SECONDS
)
except asyncio.TimeoutError:
try:
stream_mtime = stream_path.stat().st_mtime_ns
except OSError:
stream_mtime = None
if stream_mtime is not None:
record["stream_log_mtime_ns"] = stream_mtime
if stream_mtime != last_stream_mtime:
last_stream_mtime = stream_mtime
last_stream_progress_at = loop.time()
record.pop("pi_silence_inspection", None)
native_path = (
str(pi_resume_session)
if pi_resume_session is not None
else native_session_path(
spec.cli,
workspace,
record.get("session_id"),
attempt_dir,
)
)
if native_path:
record["native_session_path"] = native_path
native_mtime = native_session_mtime_ns(
record.get("native_session_path")
)
if native_mtime is not None:
record["native_session_mtime_ns"] = native_mtime
if native_mtime != last_native_mtime:
last_native_mtime = native_mtime
last_native_progress_at = loop.time()
# Native events and the separately flushed stream log
# are peer progress signals. A trailing toolResult only
# selects the timeout budget; it never overrides later
# reasoning/text output.
record["pi_activity_state"] = "working"
pi_session_state = pi_native_session_state(
record.get("native_session_path")
)
pi_phase = pi_session_state.phase
is_pi_tool_execution = pi_phase == "tool-running"
stall_timeout = (
PI_TOOL_STALL_SECONDS if is_pi_tool_execution else None
)
# Outside a toolCall->toolResult interval, model stdout/stderr
# is the liveness signal. A completed tool result changes phase
# but must not reset the model-response silence clock.
pi_inactive_seconds = loop.time() - (
max(last_native_progress_at, last_stream_progress_at)
if is_pi_tool_execution
else last_stream_progress_at
)
if spec.local_pi:
record["pi_session_phase"] = pi_phase
record["pi_session_phase_reason"] = (
pi_session_state.reason
)
record["pi_expected_tool_call_ids"] = list(
pi_session_state.expected_tool_call_ids
)
record["pi_completed_tool_call_ids"] = list(
pi_session_state.completed_tool_call_ids
)
record["pi_pending_tool_call_ids"] = list(
pi_session_state.pending_tool_call_ids
)
record["pi_stall_timeout_seconds"] = stall_timeout
record.setdefault("pi_activity_state", "starting")
if (
spec.local_pi
and not is_pi_tool_execution
and pi_inactive_seconds >= PI_MODEL_RESPONSE_STALL_SECONDS
and "pi_silence_inspection" not in record
):
inspection = {
"at": now_iso(),
"silence_seconds": round(pi_inactive_seconds, 3),
"stream_tail": log_tail_excerpt(stream_path),
}
record["pi_silence_inspection"] = inspection
diagnostic = (
f"Pi {pi_phase} stream produced no update for "
f"{pi_inactive_seconds:.1f}s; recorded stream tail for inspection "
"without terminating the model process"
)
heartbeat_log.write(f"[silence-inspection] {diagnostic}\n")
heartbeat_log.flush()
write_json(locator_path, record)
print(f"{prefix} 모델응답점검: {diagnostic}", flush=True)
if (
spec.local_pi
and is_pi_tool_execution
and not session_stall
and pi_inactive_seconds >= stall_timeout
):
session_stall = True
record["session_stall_seconds"] = round(pi_inactive_seconds, 3)
record["session_stall_native_mtime_ns"] = native_mtime
diagnostic = (
f"Pi {pi_phase} phase made no native-session or stream progress for "
f"{pi_inactive_seconds:.1f}s; terminating this process group "
"for same-model session recovery"
)
diagnostics.append(diagnostic)
diagnostic_origins.append("dispatcher:session-timeout")
heartbeat_log.write(f"[session-stall] {diagnostic}\n")
heartbeat_log.flush()
write_json(locator_path, record)
print(f"{prefix} 세션응답복구: {diagnostic}", flush=True)
await terminate_process_group(process)
continue
non_pi_inactive_seconds = loop.time() - max(
last_native_progress_at, last_stream_progress_at
)
if (
not spec.local_pi
and non_pi_inactive_seconds
>= PI_MODEL_RESPONSE_STALL_SECONDS
and "stream_silence_inspection" not in record
):
inspection = {
"at": now_iso(),
"silence_seconds": round(non_pi_inactive_seconds, 3),
"stream_tail": log_tail_excerpt(stream_path),
}
record["stream_silence_inspection"] = inspection
diagnostic = (
f"{spec.cli} emitted no stream output or native-session event for "
f"{non_pi_inactive_seconds:.1f}s; recorded stream tail for inspection "
"without terminating the model process"
)
heartbeat_log.write(f"[silence-inspection] {diagnostic}\n")
heartbeat_log.flush()
write_json(locator_path, record)
print(f"{prefix} 모델응답점검: {diagnostic}", flush=True)
heartbeat = (
f"작업중... locator={locator_path} "
f"native_session={record.get('native_session_path') or 'none'} "
f"native_mtime_ns={record.get('native_session_mtime_ns', 'none')}"
)
if spec.local_pi:
heartbeat += (
f" pi_activity={record.get('pi_activity_state')}"
f" timeout_phase={pi_phase}"
)
heartbeat_log.write(f"[heartbeat] {heartbeat}\n")
heartbeat_log.flush()
write_json(locator_path, record)
print(f"{prefix} {heartbeat}", flush=True)
continue
if raw is None:
finished_streams += 1
continue
record.pop("pi_silence_inspection", None)
record.pop("stream_silence_inspection", None)
if spec.local_pi and channel == "stdout":
record["pi_activity_state"] = "streaming"
line = raw.decode("utf-8", errors="replace").rstrip("\n")
stream_log.write(f"[{channel}] {line}\n")
stream_log.flush()
diagnostic = terminal_diagnostic(spec.cli, channel, line)
if diagnostic:
diagnostics.append(diagnostic)
diagnostic_origins.append(f"{spec.cli}:{channel}")
if spec.cli == "codex" and role == "review" and channel == "stdout":
collaboration_tool = codex_collaboration_tool(line)
if collaboration_tool and control_violation is None:
control_violation = collaboration_tool
diagnostics.append(
f"official review invoked forbidden collaboration tool: "
f"{collaboration_tool}"
)
diagnostic_origins.append("dispatcher:review-control")
print(
f"{prefix} 리뷰 제어 계약 위반: collaboration-tool="
f"{collaboration_tool}",
flush=True,
)
await terminate_process_group(process)
rendered, discovered = (
render_json_line(spec.cli, line) if channel == "stdout" else ([line], None)
)
if discovered and record.get("session_id") != discovered:
record["session_id"] = discovered
if pi_resume_session is None:
record["native_session_path"] = native_session_path(
spec.cli, workspace, discovered, attempt_dir
)
write_json(locator_path, record)
for display_line in rendered:
if display_line:
print(f"{prefix} {display_line}", flush=True)
await asyncio.gather(*readers)
return_code = await process.wait()
except asyncio.CancelledError:
for reader in readers:
reader.cancel()
if readers:
await asyncio.gather(*readers, return_exceptions=True)
await terminate_process_group(process)
runtime_error: OSError | None = None
try:
append_milestone_event(
task,
event="FINISH",
execution_id=identity,
role=role,
attempt=attempt,
model=spec.display,
result="failed:cancelled",
locator=locator_path,
)
except OSError as exc:
runtime_error = exc
record.update(
status="failed",
finished_at=now_iso(),
exit_code="cancelled",
failure_class="cancelled",
failure_source="caller-cancel",
provider_transport_failure_confirmed=False,
)
if runtime_error is not None:
record["work_log_runtime_error"] = str(runtime_error)
write_json(locator_path, record)
raise
if spec.cli == "agy":
after_agy = agy_conversations()
changed = [
path for path, mtime in after_agy.items()
if path not in before_agy or before_agy[path] != mtime
]
if changed:
selected = max(changed, key=lambda path: after_agy[path])
record["session_id"] = selected.stem
record["native_session_path"] = str(selected)
agy_diagnostics = agy_log_diagnostics(attempt_dir / "agy-cli.log")
diagnostics.extend(agy_diagnostics)
diagnostic_origins.extend("agy:cli-log" for _ in agy_diagnostics)
native_path = (
str(pi_resume_session)
if pi_resume_session is not None
else native_session_path(
spec.cli, workspace, record.get("session_id"), attempt_dir
)
)
if native_path:
record["native_session_path"] = native_path
native_mtime = native_session_mtime_ns(record.get("native_session_path"))
if native_mtime is not None:
record["native_session_mtime_ns"] = native_mtime
failure_source: str | None = None
failure_evidence: str | None = None
failure_evidence_source: str | None = None
provider_transport_failure_confirmed = False
termination = termination_signal(return_code)
if termination is not None:
record["termination_signal"] = termination[0]
record["termination_signal_inferred"] = termination[1]
if session_stall:
failure_class = "session-stall"
failure_source = "dispatcher-timeout"
failure_evidence_source = "dispatcher:session-timeout"
for index in range(len(diagnostics) - 1, -1, -1):
if diagnostic_origins[index] == failure_evidence_source:
failure_evidence = diagnostics[index]
break
record["termination_initiator"] = "dispatcher"
elif control_violation:
failure_class = "review-control-violation"
failure_source = "dispatcher-control"
failure_evidence_source = "dispatcher:review-control"
for index in range(len(diagnostics) - 1, -1, -1):
if diagnostic_origins[index] == failure_evidence_source:
failure_evidence = diagnostics[index]
break
elif return_code != 0 and termination is not None:
failure_class = "process-terminated"
failure_source = "process-termination"
record["termination_initiator"] = "unknown"
elif return_code != 0:
failure_class, failure_evidence = classify_failure_with_evidence(
"\n".join(diagnostics[-50:])
)
if failure_evidence is not None:
for index in range(len(diagnostics) - 1, -1, -1):
if diagnostics[index] == failure_evidence:
failure_evidence_source = diagnostic_origins[index]
break
if failure_class in PROVIDER_TRANSPORT_FAILURES:
failure_source = "provider-terminal-diagnostic"
provider_transport_failure_confirmed = failure_evidence is not None
elif failure_evidence is not None:
failure_source = "cli-terminal-diagnostic"
else:
failure_source = "cli-exit"
else:
failure_class = None
try:
append_milestone_event(
task,
event="FINISH",
execution_id=identity,
role=role,
attempt=attempt,
model=spec.display,
result=(
f"succeeded:0"
if return_code == 0 and failure_class is None
else f"failed:{failure_class or 'generic-error'}:{return_code}"
),
locator=locator_path,
)
except OSError as exc:
if failure_class is not None:
record["prior_failure_class"] = failure_class
record["work_log_runtime_error"] = str(exc)
failure_class = "work-log-runtime-write"
failure_source = "work-log"
failure_evidence = None
failure_evidence_source = None
provider_transport_failure_confirmed = False
if failure_evidence is not None:
record["failure_evidence_excerpt"] = failure_evidence[:FAILURE_EVIDENCE_LIMIT]
record["failure_evidence_truncated"] = (
len(failure_evidence) > FAILURE_EVIDENCE_LIMIT
)
if failure_evidence_source is not None:
record["failure_evidence_source"] = failure_evidence_source
record.update(
status="succeeded" if return_code == 0 and failure_class is None else "failed",
finished_at=now_iso(),
exit_code=return_code,
failure_class=failure_class,
failure_source=failure_source,
provider_transport_failure_confirmed=provider_transport_failure_confirmed,
)
write_json(locator_path, record)
return return_code, failure_class, locator_path
def base_prompt(task: Task, role: str, spec: AgentSpec) -> str:
if role == "review":
target = task.review or task.directory
if task.review:
return f"Read {target.resolve()} and start the review. Final in Korean."
return f"Continue the review for {target.resolve()}. Final in Korean."
if task.plan is None:
raise RuntimeError("worker PLAN이 없다")
target = task.plan.resolve()
if role == "selfcheck":
if task.review is None:
raise RuntimeError("selfcheck CODE_REVIEW 파일이 없다")
return (
f"Think in English. Final in Korean. Read {task.review.resolve()} and fill "
"every missing implementation field. Do not finish until all implementation "
"fields are complete. This is a self-check of completed work, not a review. "
f"Read {target} and finish any missing work. Recheck and fix your work."
)
if spec.local_pi:
return f"Think in English. Final in Korean. Read {target} and complete the task."
return f"Read {target} and complete the task. Final in Korean."
def continuation_prompt(
task: Task,
role: str,
locator: Path,
*,
local_pi: bool = False,
resume_same_pi_session: bool = False,
) -> str:
if local_pi:
if resume_same_pi_session:
return (
"Think in English. Final in Korean. Continue this session and complete "
"the current task."
)
if role == "selfcheck" and task.plan and task.review:
return (
f"Think in English. Final in Korean. Read {task.review.resolve()} and fill "
"every missing implementation field. Do not finish until all implementation "
"fields are complete. This is a self-check of completed work, not a review. "
f"Read {task.plan.resolve()} and finish any missing work. Recheck and fix "
"your work."
)
target = task.plan or task.directory
return f"Think in English. Final in Korean. Read {target.resolve()} and complete the task."
if role == "review":
return f"Continue the review for {task.directory.resolve()}. Final in Korean."
return (
f"Continue from {locator.resolve()}. Check the saved context and current "
"workspace. Final in Korean."
)
async def run_escalating(
workspace: Path,
store: StateStore,
task: Task,
role: str,
initial: AgentSpec,
invocation_semaphore: asyncio.Semaphore | None = None,
initial_resume_locator: Path | None = None,
) -> tuple[bool, Path | None]:
spec = initial
previous_locator = initial_resume_locator
codex_recovery_count = 0
codex_session_stall_retries = 0
review_control_retries = 0
pi_recovery_retries = 0
generic_retries = 0
terminal_recovery_retries = 0
pi_resume_locator = initial_resume_locator
recovery_failures = 0
if isinstance(store, StateStore):
persisted = store.task_state(task).get("recovery_failures", {})
if isinstance(persisted, dict):
recovery_failures = int(persisted.get(role, 0))
if recovery_failures >= RECOVERY_FAILURE_LIMIT:
locator = initial_resume_locator
reason = (
f"{role} recovery failure limit already exhausted: "
f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}"
)
if isinstance(store, StateStore):
store.update_task(task, blocked=f"{reason} locator={locator}")
banner(
"작업차단",
task.name,
[
"reason=recovery-failure-limit",
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
f"locator={locator}",
],
)
return False, locator
while True:
prompt = (
base_prompt(task, role, spec)
if previous_locator is None
else continuation_prompt(
task,
role,
previous_locator,
local_pi=spec.local_pi,
resume_same_pi_session=pi_resume_locator is not None,
)
)
if invocation_semaphore is None:
rc, failure, locator = await invoke(
workspace,
store,
task,
role,
spec,
prompt,
resume_locator=pi_resume_locator,
)
else:
async with invocation_semaphore:
rc, failure, locator = await invoke(
workspace,
store,
task,
role,
spec,
prompt,
resume_locator=pi_resume_locator,
)
pi_resume_locator = None
if rc == 0 and failure is None:
if isinstance(store, StateStore):
state = store.task_state(task)
persisted = dict(state.get("recovery_failures", {}))
persisted.pop(role, None)
store.update_task(task, recovery_failures=persisted)
return True, locator
failure = failure or "generic-error"
if failure in {
"work-log-blocked",
"work-log-incomplete",
"work-log-setup",
"work-log-runtime-write",
}:
banner("작업차단", task.name, failure_report_lines(failure, locator))
return False, locator
recovery_failures += 1
if isinstance(store, StateStore):
state = store.task_state(task)
persisted = dict(state.get("recovery_failures", {}))
persisted[role] = recovery_failures
store.update_task(task, recovery_failures=persisted)
if recovery_failures >= RECOVERY_FAILURE_LIMIT:
reason = (
f"{role} recovery failure limit exhausted: "
f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}"
)
if isinstance(store, StateStore):
store.update_task(task, blocked=f"{reason} locator={locator}")
banner(
"작업차단",
task.name,
[
"reason=recovery-failure-limit",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
return False, locator
if role == "review" and failure == "review-control-violation":
review_control_retries += 1
banner(
"리뷰재시도",
task.name,
[
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = None
await asyncio.sleep(min(30, 2 ** min(review_control_retries, 5)))
continue
if spec.local_pi:
if (
spec.model.startswith("laguna-s")
and failure in {"context-limit", "session-stall"}
):
pi_recovery_retries += 1
banner(
"Pi세션연속재시작",
task.name,
[
f"model={spec.display}",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = locator
pi_resume_locator = locator
await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5)))
continue
pi_recovery_retries += 1
if failure == "session-stall":
event = "세션응답복구재시도"
elif failure in {
"provider-connection",
"provider-stream-disconnect",
}:
event = "세션연결재시도"
else:
event = "Pi복구재시도"
banner(
event,
task.name,
[
f"model={spec.display}",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = locator
await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5)))
continue
if spec.cli == "codex" and failure == "session-stall":
codex_session_stall_retries += 1
banner(
"세션응답복구재시도",
task.name,
[
f"model={spec.display}",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = locator
await asyncio.sleep(min(30, 2 ** min(codex_session_stall_retries, 5)))
continue
if failure == "generic-error":
generic_retries += 1
banner(
"작업복구재시도",
task.name,
[
f"model={spec.display}",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = locator
await asyncio.sleep(min(30, 2 ** min(generic_retries, 5)))
continue
next_spec = promoted_spec(spec, codex_recovery_count)
if next_spec is None:
terminal_recovery_retries += 1
banner(
"모델복구재시도",
task.name,
[
f"model={spec.display}",
*failure_report_lines(failure, locator),
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
],
)
previous_locator = locator
await asyncio.sleep(min(30, 2 ** min(terminal_recovery_retries, 5)))
continue
if spec.cli == "codex":
codex_recovery_count += 1
banner(
"모델승격",
task.name,
[
f"from={spec.display}",
f"to={next_spec.display}",
*failure_report_lines(failure, locator),
],
)
spec = next_spec
previous_locator = locator
def task_signature(workspace: Path, task: Task) -> str:
digest = hashlib.sha256()
if not task.directory.exists():
return "moved"
for path in sorted(p for p in task.directory.iterdir() if p.is_file()):
if (
PLAN_RE.match(path.name)
or REVIEW_RE.match(path.name)
or path.name.endswith(".log")
):
digest.update(path.name.encode())
digest.update(sha256_file(path).encode())
for raw_path in sorted(task.write_set):
path = Path(raw_path)
path = path if path.is_absolute() else workspace / path
digest.update(raw_path.encode())
if path.is_file():
digest.update(str(path.stat().st_mode).encode())
digest.update(sha256_file(path).encode())
elif path.exists():
digest.update(b"non-file")
else:
digest.update(b"missing")
return digest.hexdigest()
def read_verdict(path: Path) -> str | None:
if not path.exists():
return None
return verdict_from_text(path.read_text(encoding="utf-8", errors="replace"))
def verdict_from_text(text: str) -> str | None:
headings = list(VERDICT_HEADING_RE.finditer(text))
if not headings:
return None
heading = headings[-1]
next_heading = re.search(r"^##\s+", text[heading.end():], re.MULTILINE)
end = heading.end() + next_heading.start() if next_heading else len(text)
section = text[heading.end():end]
inline_matches = list(VERDICT_LINE_RE.finditer(section))
block_matches = list(VERDICT_BLOCK_RE.finditer(section))
matches = inline_matches + block_matches
return matches[0].group(1) if len(matches) == 1 else None
def matching_archive_directories_by_name(
workspace: Path,
task_name: str,
*,
require_complete: bool = True,
) -> list[Path]:
archive = workspace / "agent-task" / "archive"
parts = task_name.split("/")
if not archive.is_dir() or len(parts) not in {1, 2}:
return []
group = parts[0]
final_name = parts[-1]
suffix_re = re.compile(rf"^{re.escape(final_name)}(?:_\d+)?$")
matches: list[Path] = []
for year in archive.iterdir():
if not year.is_dir():
continue
for month in year.iterdir():
if not month.is_dir():
continue
parent = month if len(parts) == 1 else month / group
if not parent.is_dir():
continue
for candidate in parent.iterdir():
if (
candidate.is_dir()
and suffix_re.match(candidate.name)
and (
not require_complete
or (candidate / "complete.log").is_file()
)
):
matches.append(candidate)
return sorted(matches)
def matching_archive_directories(workspace: Path, task: Task) -> list[Path]:
return matching_archive_directories_by_name(workspace, task.name)
def cleanup_completed_task_attempt_logs(runs: Path, task_name: str) -> int:
"""Remove only dispatcher-owned logs for a task after its complete archive exists."""
removed = 0
if not runs.is_dir():
return removed
for attempt_dir in runs.iterdir():
if not attempt_dir.is_dir():
continue
locator = attempt_dir / "locator.json"
try:
record = json.loads(locator.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
if record.get("task") != task_name:
continue
shutil.rmtree(attempt_dir)
removed += 1
return removed
def review_fingerprints(workspace: Path, task: Task) -> set[tuple[str, str]]:
directories = [task.directory] if task.directory.is_dir() else []
directories.extend(matching_archive_directories(workspace, task))
fingerprints: set[tuple[str, str]] = set()
for directory in directories:
for path in directory.iterdir():
if path.is_file() and (path.name.startswith("code_review_") or REVIEW_RE.match(path.name)):
fingerprints.add((str(path.resolve()), sha256_file(path)))
return fingerprints
def review_outcome(
workspace: Path, task: Task, prior_fingerprints: set[tuple[str, str]]
) -> dict[str, str]:
if task.directory.is_dir():
directories = [task.directory]
archives: list[Path] = []
else:
archives = matching_archive_directories(workspace, task)
directories = list(archives)
newest_directory: Path | None = None
newest_log: Path | None = None
newest_mtime = -1
for directory in directories:
logs = list(directory.glob("code_review_*.log"))
logs.extend(path for path in directory.iterdir() if path.is_file() and REVIEW_RE.match(path.name))
for log in logs:
mtime = log.stat().st_mtime_ns
fingerprint = (str(log.resolve()), sha256_file(log))
if fingerprint not in prior_fingerprints and mtime > newest_mtime and read_verdict(log):
newest_directory = directory
newest_log = log
newest_mtime = mtime
verdict = read_verdict(newest_log) if newest_log else "UNKNOWN"
if newest_directory in archives:
state = "archived"
elif newest_directory and (newest_directory / "USER_REVIEW.md").exists():
state = "user-review"
elif newest_directory and (newest_directory / "complete.log").exists():
state = "complete-finalization"
elif newest_log and REVIEW_RE.match(newest_log.name):
state = "finalization-pending"
elif newest_directory and any(REVIEW_RE.match(path.name) for path in newest_directory.iterdir() if path.is_file()):
state = "follow-up"
else:
state = "changed"
return {
"verdict": verdict or "UNKNOWN",
"state": state,
"path": str(newest_directory or task.directory),
"review_log": str(newest_log) if newest_log else "unknown",
}
async def run_worker(
workspace: Path,
store: StateStore,
task: Task,
semaphores: dict[str, asyncio.Semaphore],
resume_locator: Path | None = None,
) -> None:
spec = route_agent(task)
resource = f"pi:{spec.model}" if spec.cli == "pi" else spec.cli
work_log = milestone_work_log_path(task)
banner(
"작업시작",
task.name,
[
f"model={spec.display}",
f"plan={task.plan.resolve()}",
f"work_log={work_log.resolve()}",
],
)
success, locator = await run_escalating(
workspace,
store,
task,
"worker",
spec,
invocation_semaphore=semaphores[resource],
initial_resume_locator=resume_locator,
)
if not success:
current = store.task_state(task).get("blocked")
store.update_task(
task, blocked=current or f"worker failure locator={locator}"
)
return
store.update_task(
task,
worker_done=True,
worker_cli=spec.cli,
worker_model=spec.model,
selfcheck_done=not spec.local_pi,
blocked=None,
)
async def run_selfcheck(
workspace: Path,
store: StateStore,
task: Task,
semaphores: dict[str, asyncio.Semaphore],
resume_locator: Path | None = None,
) -> None:
spec = route_agent(task)
if not spec.local_pi:
raise RuntimeError("Pi가 아닌 route에 selfcheck stage가 배정됐다")
resource = f"pi:{spec.model}"
work_log = milestone_work_log_path(task)
banner(
"자가검증시작",
task.name,
[
f"model={spec.display}",
f"plan={task.plan.resolve()}",
f"work_log={work_log.resolve()}",
],
)
retry = 0
if isinstance(store, StateStore):
retry = int(store.task_state(task).get("selfcheck_incomplete", 0))
if retry >= SELF_CHECK_INCOMPLETE_LIMIT:
locator = resume_locator
reason = (
"selfcheck implementation field limit already exhausted: "
f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}"
)
store.update_task(task, blocked=f"{reason} locator={locator}")
banner(
"작업차단",
task.name,
[
"reason=selfcheck-incomplete-limit",
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
f"locator={locator}",
],
)
return
while True:
success, locator = await run_escalating(
workspace,
store,
task,
"selfcheck",
spec,
invocation_semaphore=semaphores[resource],
initial_resume_locator=resume_locator,
)
resume_locator = None
if not success:
current = store.task_state(task).get("blocked")
store.update_task(
task, blocked=current or f"selfcheck failure locator={locator}"
)
return
errors = implementation_review_errors(task)
if not errors:
break
retry += 1
if retry >= SELF_CHECK_INCOMPLETE_LIMIT:
reason = (
"selfcheck implementation fields remain incomplete: "
f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}"
)
store.update_task(
task,
blocked=f"{reason} locator={locator}",
selfcheck_incomplete=retry,
)
banner(
"작업차단",
task.name,
[
"reason=selfcheck-incomplete-limit",
f"detail={'; '.join(errors)}",
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
f"locator={locator}",
],
)
return
store.update_task(task, selfcheck_incomplete=retry)
banner(
"자가검증재시도",
task.name,
[
f"reason={'; '.join(errors)}",
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
f"locator={locator}",
],
)
store.update_task(
task,
selfcheck_done=True,
selfcheck_incomplete=0,
blocked=None,
)
async def run_review(
workspace: Path,
store: StateStore,
task: Task,
) -> str | None:
spec = AgentSpec("codex", "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh")
state = store.task_state(task)
prior_no_progress = int(state.get("review_no_progress", 0))
if prior_no_progress >= REVIEW_NO_PROGRESS_LIMIT:
locator = state.get("active_locator")
reason = (
"review no-progress limit already exhausted: "
f"{prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}"
)
store.update_task(task, blocked=f"{reason} locator={locator}")
banner(
"작업차단",
task.name,
[
"reason=review-no-progress-limit",
f"unchanged_review_attempts={prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}",
f"locator={locator}",
],
)
return None
before = task_signature(workspace, task)
prior_review_fingerprints = review_fingerprints(workspace, task)
target = task.review.resolve() if task.review else task.directory.resolve()
banner("리뷰시작", task.name, [f"model={spec.display}", f"review={target}"])
success, locator = await run_escalating(workspace, store, task, "review", spec)
if not success:
current = store.task_state(task).get("blocked")
store.update_task(
task, blocked=current or f"review failure locator={locator}"
)
return None
after = task_signature(workspace, task)
if before == after:
state = store.task_state(task)
count = int(state.get("review_no_progress", 0)) + 1
if count >= REVIEW_NO_PROGRESS_LIMIT:
reason = (
"review made no progress: "
f"{count}/{REVIEW_NO_PROGRESS_LIMIT} locator={locator}"
)
store.update_task(
task,
review_no_progress=count,
blocked=reason,
)
banner(
"작업차단",
task.name,
[
"reason=review-no-progress-limit",
f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}",
f"locator={locator}",
],
)
return None
store.update_task(task, review_no_progress=count)
banner(
"루프정체경고",
task.name,
[
f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}",
f"locator={locator}",
],
)
await asyncio.sleep(min(30, count * 5))
else:
store.update_task(task, review_no_progress=0, blocked=None)
outcome = review_outcome(workspace, task, prior_review_fingerprints)
banner(
"리뷰결과",
task.name,
[
f"verdict={outcome['verdict']}",
f"state={outcome['state']}",
f"path={outcome['path']}",
f"review_log={outcome['review_log']}",
f"locator={locator}",
],
)
if outcome["verdict"] == "PASS" and outcome["state"] == "archived":
banner("작업완료", task.name, [f"archive={outcome['path']}", f"locator={locator}"])
return outcome["path"]
if outcome["verdict"] == "UNKNOWN" or outcome["state"] == "changed":
raise RuntimeError(
"official review가 판정과 다음 파일 상태를 materialize하지 않았다; "
f"locator={locator}"
)
if outcome["state"] == "archived":
raise RuntimeError(
f"PASS가 아닌 review가 완료 archive로 이동했다: "
f"verdict={outcome['verdict']} locator={locator}"
)
return None
def status_lines(task: Task, stage: str, dependency: str) -> list[str]:
route = f"{task.lane}-G{task.grade:02d}" if task.lane and task.grade else "recovery"
return [f"stage={stage}", f"route={route}", f"dependency={dependency}"]
def select_dispatch_candidates(
ready: list[tuple[Task, str]],
) -> tuple[list[tuple[Task, str]], list[tuple[Task, str]], str]:
ready_reviews = [(task, stage) for task, stage in ready if stage == "review"]
ready_workers = [(task, stage) for task, stage in ready if stage in {"worker", "selfcheck"}]
# Each scheduler pass admits every dependency-ready task. The running map
# prevents only duplicate attempts for the same task; phases and write-sets
# never impose a cross-task barrier.
return ready_reviews + ready_workers, [], ""
def ensure_review_shared_state(workspace: Path) -> None:
helper = workspace / "agent-ops" / "bin" / "ai-ignore.sh"
if not helper.is_file():
raise RuntimeError(f"review shared-state helper가 없다: {helper}")
command = [
"bash",
"-c",
'source "$1" && agent_ops_ensure_gitignore_task_artifact_block "$2"',
"agent-task-review-preflight",
str(helper),
str(workspace / ".gitignore"),
]
completed = subprocess.run(
command,
cwd=workspace,
capture_output=True,
text=True,
check=False,
)
if completed.returncode != 0:
diagnostic = (completed.stderr or completed.stdout).strip()
raise RuntimeError(
f"review shared-state preflight 실패: {diagnostic or completed.returncode}"
)
async def dispatch(args: argparse.Namespace) -> int:
workspace = Path(args.workspace).resolve()
store = StateStore(workspace)
try:
return await dispatch_with_store(args, workspace, store)
finally:
store.close()
async def dispatch_with_store(
args: argparse.Namespace,
workspace: Path,
store: StateStore,
) -> int:
orchestration_scope = args.task_group or "__all__"
if args.retry_blocked and not args.dry_run:
store.clear_blocked(args.task_group)
semaphores = {
"pi:ornith-fast": asyncio.Semaphore(3),
"pi:laguna-s:2.1": asyncio.Semaphore(2),
"agy": asyncio.Semaphore(1),
"claude": asyncio.Semaphore(64),
"codex": asyncio.Semaphore(64),
}
running: dict[str, asyncio.Task[str | None]] = {}
last_wait: dict[str, str] = {}
completed_tasks: dict[str, str] = {}
fatal_errors: dict[str, str] = {}
review_shared_state_ready = False
candidate_scope: set[str] | None = None
task_cache: dict[str, Task] | None = None
resume_locators: dict[str, Path] = {}
while True:
if task_cache is None:
tasks = scan_tasks(workspace, args.task_group)
task_cache = {task.name: task for task in tasks}
else:
tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name))
if args.dry_run:
persistent_errors: dict[str, str] = {}
observed_tasks: set[str] = set()
else:
store.prepare_orchestration(orchestration_scope, tasks, workspace)
active_or_running = {task.name for task in tasks} | set(running)
reconciled_completed, persistent_errors = store.reconcile_orchestration(
orchestration_scope,
workspace,
active_or_running,
)
completed_tasks.update(reconciled_completed)
observed_tasks = store.orchestration_tasks(orchestration_scope)
if not tasks and not running:
incomplete = sorted(observed_tasks - completed_tasks.keys())
if incomplete or fatal_errors:
details = [
*(f"incomplete={name}" for name in incomplete),
*(
f"persistent[{name}]={reason}"
for name, reason in sorted(persistent_errors.items())
),
*(f"error[{name}]={reason}" for name, reason in sorted(fatal_errors.items())),
]
if not args.dry_run:
store.mark_orchestration_blocked(
orchestration_scope,
{
name: (
"blocked",
persistent_errors.get(name)
or fatal_errors.get(name)
or "관찰된 task가 완료되지 않았다",
)
for name in set(incomplete) | set(fatal_errors)
},
)
banner("디스패치차단", args.task_group or "agent-task", details)
return 2
if not args.dry_run:
store.mark_orchestration_complete(orchestration_scope)
banner(
"작업완료",
args.task_group or "agent-task",
[
"active task 없음",
f"verified_complete_tasks={len(completed_tasks)}",
*(f"complete[{name}]={path}" for name, path in sorted(completed_tasks.items())),
],
)
return 0
task_by_name = {task.name: task for task in tasks}
finished_names: set[str] = set()
complete_log_created = False
for name, future in list(running.items()):
if not future.done():
continue
finished_names.add(name)
completed_archive: str | None = None
try:
completed_archive = future.result()
if completed_archive:
complete_log_created = True
store.mark_orchestration_task_complete(
orchestration_scope, name, completed_archive
)
completed_tasks[name] = completed_archive
except Exception as exc: # keep other independent tasks alive
banner("작업차단", name, [f"dispatcher exception={exc}"])
fatal_errors[name] = str(exc)
task = task_by_name.get(name)
if task:
store.update_task(task, blocked=f"dispatcher exception={exc}")
task = task_by_name.get(name)
if task:
store.clear_active(task)
if not completed_archive:
refreshed = read_task_directory(workspace, task.directory)
if refreshed is None:
task_cache.pop(name, None)
else:
task_cache[name] = refreshed
if completed_archive:
task_cache.pop(name, None)
del running[name]
if finished_names:
if complete_log_created:
tasks = scan_tasks(workspace, args.task_group)
task_cache = {task.name: task for task in tasks}
store.prepare_orchestration(orchestration_scope, tasks, workspace)
active_or_running = {task.name for task in tasks} | set(running)
reconciled_completed, persistent_errors = store.reconcile_orchestration(
orchestration_scope,
workspace,
active_or_running,
)
completed_tasks.update(reconciled_completed)
observed_tasks = store.orchestration_tasks(orchestration_scope)
candidate_scope = None
else:
tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name))
candidate_scope = finished_names
if not tasks and not running:
# A completion-triggered full scan may have removed the last active task.
continue
ready: list[tuple[Task, str]] = []
waiting_tasks: list[str] = []
externally_active: list[tuple[Task, str]] = []
blocked_details: dict[str, tuple[str, str, str]] = {}
for task in tasks:
state = store.peek_task_state(task) if args.dry_run else store.task_state(task)
dependency_ready, dependency = dependency_state(workspace, task)
stage = task_stage(task, state)
if state.get("active_stage") and task.name not in running:
active_stage = str(state["active_stage"])
active_live, active_detail = external_active_is_live(state)
if active_live:
reason = f"외부 실행중: stage={active_stage}; {active_detail}"
active_key = (
f"active|{active_stage}|{state.get('active_locator') or 'unknown'}"
)
externally_active.append((task, active_stage))
blocked_details[task.name] = ("작업중", stage, reason)
if not args.dry_run and last_wait.get(task.name) != active_key:
banner("작업중", task.name, status_lines(task, stage, reason))
last_wait[task.name] = active_key
continue
if not args.dry_run:
resume_locator = laguna_resume_locator(state)
if resume_locator is not None:
resume_locators[task.name] = resume_locator
banner(
"작업복구",
task.name,
status_lines(task, stage, f"stale active 제외: {active_detail}"),
)
store.clear_active(task)
state = store.task_state(task)
reason = ""
if task.errors:
reason = "; ".join(task.errors)
elif task.user_review:
blocking, detail = user_review_blocker_state(task.user_review)
if task.plan is not None or task.review is not None:
blocking = False
detail = "active PLAN/CODE_REVIEW와 공존한다"
reason = (
f"USER_REVIEW 대기: {task.user_review}; {detail}"
if blocking
else (
"USER_REVIEW stop 계약 불충족: "
f"{task.user_review}; {detail}"
)
)
elif state.get("blocked"):
reason = str(state["blocked"])
elif not dependency_ready:
reason = dependency
if reason:
wait_key = f"{stage}|{reason}"
event = (
"작업차단"
if stage in {"blocked", "user-review"} or state.get("blocked")
else "작업대기"
)
blocked_details[task.name] = (event, stage, reason)
if not args.dry_run and last_wait.get(task.name) != wait_key:
banner(event, task.name, status_lines(task, stage, reason))
last_wait[task.name] = wait_key
waiting_tasks.append(task.name)
continue
if task.name not in running and (
candidate_scope is None or task.name in candidate_scope
):
ready.append((task, stage))
if args.dry_run:
ready_by_name = {task.name: stage for task, stage in ready}
for task in tasks:
if task.name in ready_by_name:
stage = ready_by_name[task.name]
spec = (
AgentSpec("codex", "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh")
if stage == "review"
else route_agent(task)
)
banner("작업대기", task.name, status_lines(task, stage, "ready") + [f"model={spec.display}"])
else:
event, stage, reason = blocked_details[task.name]
banner(event, task.name, status_lines(task, stage, reason))
return 2 if waiting_tasks and not ready else 0
candidates, deferred, phase_wait_reason = select_dispatch_candidates(ready)
for task, stage in deferred:
wait_key = f"{stage}|{phase_wait_reason}"
if last_wait.get(task.name) != wait_key:
banner("작업대기", task.name, status_lines(task, stage, phase_wait_reason))
last_wait[task.name] = wait_key
if (
not review_shared_state_ready
and any(stage == "review" for _, stage in candidates)
):
ensure_review_shared_state(workspace)
review_shared_state_ready = True
scheduled = False
for task, stage in candidates:
store.mark_active(task, stage)
resume_locator = resume_locators.pop(task.name, None)
if stage == "review":
future = asyncio.create_task(run_review(workspace, store, task))
elif stage == "selfcheck":
future = asyncio.create_task(
run_selfcheck(
workspace,
store,
task,
semaphores,
**(
{"resume_locator": resume_locator}
if resume_locator is not None
else {}
),
)
)
else:
future = asyncio.create_task(
run_worker(
workspace,
store,
task,
semaphores,
**(
{"resume_locator": resume_locator}
if resume_locator is not None
else {}
),
)
)
running[task.name] = future
last_wait.pop(task.name, None)
scheduled = True
if running:
await asyncio.wait(running.values(), return_when=asyncio.FIRST_COMPLETED)
continue
if externally_active:
banner(
"디스패치추적대기",
args.task_group or "agent-task",
["새 실행 후보 없음", "active task는 caller가 계속 추적"],
)
return 3
if not scheduled:
store.mark_orchestration_blocked(
orchestration_scope,
{
name: (
"blocked" if event == "작업차단" else "waiting",
detail,
)
for name, (event, _, detail) in blocked_details.items()
},
)
reason = f"waiting={','.join(sorted(waiting_tasks))}"
banner(
"디스패치차단",
args.task_group or "agent-task",
[
"실행 가능한 독립 작업을 모두 소진함",
reason,
f"verified_complete_tasks={len(completed_tasks)}",
*(
f"complete[{name}]={path}"
for name, path in sorted(completed_tasks.items())
),
*(
f"{name}: stage={stage}; reason={detail}"
for name, (_, stage, detail) in sorted(
blocked_details.items()
)
),
],
)
return 2
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workspace", default=".", help="repository root (default: current directory)")
parser.add_argument("--task-group", help="run only agent-task/<task_group>")
parser.add_argument("--dry-run", action="store_true", help="classify and print without launching CLIs")
parser.add_argument("--retry-blocked", action="store_true", help="clear dispatcher-local blocked state")
return parser.parse_args()
def main() -> int:
args = parse_args()
try:
return asyncio.run(dispatch(args))
except KeyboardInterrupt:
print("\n중단됨", file=sys.stderr)
return 130
except RuntimeError as exc:
print(f"dispatcher error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())