toki
de4d8f4ff8
세 Agent의 direct route를 동일한 fail-closed preflight와 격리 실행 경계에서 비교하고, 관측되지 않은 preset 셀이 실행되는 것을 막기 위해 연결 계약과 증거 수집 흐름을 고정한다.
190 lines
4.9 KiB
Python
190 lines
4.9 KiB
Python
"""
|
|
agent_benchmark - Closed manifest loader, validator, and workspace preparation.
|
|
|
|
This package is standard-library-only and exposes stable immutable
|
|
manifest and workspace preparation APIs consuming only frozen inputs.
|
|
All returned objects are frozen dataclasses.
|
|
"""
|
|
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
ExpectedBinding,
|
|
Fixture,
|
|
IopCell,
|
|
Manifest,
|
|
ManifestDigestError,
|
|
ManifestError,
|
|
ManifestPathError,
|
|
ManifestValidationError,
|
|
MatrixCell,
|
|
Timeout,
|
|
Viewport,
|
|
digest_manifest_and_resolved_inputs,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
validate_manifest_bytes,
|
|
)
|
|
from scripts.agent_benchmark.workspace import (
|
|
AttemptIdentity,
|
|
PreparedWorkspace,
|
|
TestbedError,
|
|
TestbedProvenance,
|
|
WorkspaceChecksumError,
|
|
WorkspaceError,
|
|
WorkspacePathError,
|
|
WorkspaceValidationError,
|
|
inspect_testbed_provenance,
|
|
prepare_workspace,
|
|
validate_attempt_identity,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
COMPLETION_STOP_AFTER_IDLE,
|
|
SUBMISSION_ARGV_TASK,
|
|
SUBMISSION_STDIN_ONCE,
|
|
CancellationToken,
|
|
CaptureStream,
|
|
InvocationResult,
|
|
InvocationSpec,
|
|
LifecycleError,
|
|
LifecycleProtocolError,
|
|
LifecycleRecoveryError,
|
|
LifecycleValidationError,
|
|
SupervisorLocator,
|
|
TerminalOutcome,
|
|
env_pairs,
|
|
exact_value_redactor,
|
|
read_locator,
|
|
recover_invocation,
|
|
run_invocation,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
CallerCapability,
|
|
ConnectivityEvidenceError,
|
|
ConnectivityIssue,
|
|
ConnectivityResult,
|
|
ConnectivityValidationError,
|
|
EffectiveBinding,
|
|
RequestedEffectiveBinding,
|
|
canonical_evidence_bytes,
|
|
classify_issues,
|
|
make_result,
|
|
read_evidence,
|
|
validate_result,
|
|
write_evidence,
|
|
)
|
|
from scripts.agent_benchmark.claude_iop import (
|
|
ClaudeIopAdapter,
|
|
ClaudeIopRuntime,
|
|
claude_capability,
|
|
)
|
|
from scripts.agent_benchmark.agy_iop import (
|
|
AgyCapability,
|
|
AgyEventParser,
|
|
AgyPreflightResult,
|
|
AgyRuntimeInputs,
|
|
AgyRuntimeObservation,
|
|
inspect_agy_iop_capability,
|
|
preflight_agy_iop,
|
|
run_agy_invocation,
|
|
)
|
|
from scripts.agent_benchmark.codex_iop import (
|
|
CodexInvocation,
|
|
CodexInvocationResult,
|
|
CodexJSONLParser,
|
|
CodexRuntime,
|
|
build_codex_invocation,
|
|
codex_capability,
|
|
run_codex_invocation,
|
|
runtime_from_environment,
|
|
)
|
|
from scripts.agent_benchmark.attempts import (
|
|
Attempt, AttemptError, AttemptStateError, CapabilityUnavailable, RunBusyError,
|
|
PreflightAdapter, PreflightObservation, RunIdentity, RunPathError, RunStore,
|
|
Slot, collect_preflight_observations, preflight_manifest, run_slots,
|
|
)
|
|
|
|
__all__ = [
|
|
"Manifest",
|
|
"Timeout",
|
|
"Viewport",
|
|
"AssetMapping",
|
|
"Fixture",
|
|
"IopCell",
|
|
"ExpectedBinding",
|
|
"MatrixCell",
|
|
"ManifestError",
|
|
"ManifestValidationError",
|
|
"ManifestPathError",
|
|
"ManifestDigestError",
|
|
"load_manifest",
|
|
"validate_manifest_bytes",
|
|
"digest_workspace_inputs",
|
|
"digest_manifest_and_resolved_inputs",
|
|
"AttemptIdentity",
|
|
"TestbedProvenance",
|
|
"PreparedWorkspace",
|
|
"WorkspaceError",
|
|
"WorkspaceValidationError",
|
|
"WorkspacePathError",
|
|
"WorkspaceChecksumError",
|
|
"TestbedError",
|
|
"prepare_workspace",
|
|
"inspect_testbed_provenance",
|
|
"validate_attempt_identity",
|
|
"SUBMISSION_ARGV_TASK",
|
|
"SUBMISSION_STDIN_ONCE",
|
|
"COMPLETION_EXIT_AFTER_IDLE",
|
|
"COMPLETION_STOP_AFTER_IDLE",
|
|
"InvocationSpec",
|
|
"InvocationResult",
|
|
"CaptureStream",
|
|
"SupervisorLocator",
|
|
"TerminalOutcome",
|
|
"CancellationToken",
|
|
"LifecycleError",
|
|
"LifecycleValidationError",
|
|
"LifecycleProtocolError",
|
|
"LifecycleRecoveryError",
|
|
"run_invocation",
|
|
"recover_invocation",
|
|
"read_locator",
|
|
"env_pairs",
|
|
"exact_value_redactor",
|
|
"CallerCapability",
|
|
"ConnectivityEvidenceError",
|
|
"ConnectivityIssue",
|
|
"ConnectivityResult",
|
|
"ConnectivityValidationError",
|
|
"EffectiveBinding",
|
|
"RequestedEffectiveBinding",
|
|
"canonical_evidence_bytes",
|
|
"classify_issues",
|
|
"make_result",
|
|
"read_evidence",
|
|
"validate_result",
|
|
"write_evidence",
|
|
"ClaudeIopAdapter",
|
|
"ClaudeIopRuntime",
|
|
"claude_capability",
|
|
"AgyCapability",
|
|
"AgyEventParser",
|
|
"AgyPreflightResult",
|
|
"AgyRuntimeInputs",
|
|
"AgyRuntimeObservation",
|
|
"inspect_agy_iop_capability",
|
|
"preflight_agy_iop",
|
|
"run_agy_invocation",
|
|
"CodexInvocation",
|
|
"CodexInvocationResult",
|
|
"CodexJSONLParser",
|
|
"CodexRuntime",
|
|
"build_codex_invocation",
|
|
"codex_capability",
|
|
"run_codex_invocation",
|
|
"runtime_from_environment",
|
|
"Attempt", "AttemptError", "AttemptStateError", "CapabilityUnavailable",
|
|
"RunBusyError", "PreflightAdapter", "PreflightObservation", "RunIdentity",
|
|
"RunPathError", "RunStore", "Slot", "collect_preflight_observations",
|
|
"preflight_manifest", "run_slots",
|
|
]
|