#!/usr/bin/env bash # scripts/e2e-hot-path-agents.sh # # Secret-safe Claude/Pi Hot Path smoke harness. # # Modes: # --self-test Credential-free behavioral oracle. Builds fake Claude/Pi # binaries, a fake Edge binary/config, a fake Pi config dir, # runtime identity evidence, a live observation log, disposable # workspaces and sentinel secrets under one mktemp -d, then # runs the fixed 2x5 matrix through the same manifest builder # and validator used by --run and asserts every safety proof, # including runtime/profile binding and fresh-observation # rejection. # --preflight-only Validate non-secret inputs, current worktree fingerprint, # Edge/Pi/CLI runtime identity, base/profile/alias binding and # the observation log without invoking any agent. # --run Validate inputs/identity, bind both CLIs to the supplied IOP # base/profile and per-scenario preset alias, run the fixed # {claude,pi} x {direct,light-pass,repair,write-unavailable, # timeout-cancel} matrix in disposable workspaces while # capturing only freshly appended observation-log records, and # atomically emit a redacted caller-supplied manifest. # # This harness never prints secret, endpoint, config, or model values. Missing or # mismatched source/worktree/runtime/config/binary/fixture/base/profile/alias # facts exit 69 before any agent invocation. Each case consumes only observation # records appended by the selected runtime after that case started; stale, # rotated, truncated, missing, mixed, or wrong-stage evidence is rejected. No # Makefile, deployment, shared-process, or tracked smoke output is touched. The # self-test path does not contact the network and does not invoke the installed # Pi/Claude/Edge or any provider. set -euo pipefail readonly EXIT_OK=0 readonly EXIT_USAGE=64 readonly EXIT_VALIDATION=69 readonly EXIT_SOFTWARE=70 readonly SCHEMA_VERSION="1" readonly EXIT_TIMEOUT=124 # Exact pinned adapter argv (the prompt and provider/model identity are appended # by the adapter builders; the base/model is bound through the environment and is # never serialized). readonly CLAUDE_FLAGS=(--print --output-format stream-json --include-partial-messages --no-session-persistence --bare) readonly PI_FLAGS=(--provider --model --mode json --print --no-session) readonly AGENTS=(claude pi) readonly SCENARIOS=(direct light-pass repair write-unavailable timeout-cancel) readonly EXPECTED_CASE_IDS=( claude:direct claude:light-pass claude:repair claude:write-unavailable claude:timeout-cancel pi:direct pi:light-pass pi:repair pi:write-unavailable pi:timeout-cancel ) # Deterministic worktree fingerprint input set (SDD S16 runtime/source identity). # A content change to any of these paths changes the fingerprint without exposing # any file value. readonly WORKTREE_FINGERPRINT_PATHS=( apps/edge packages/go/streamgate packages/go/config scripts/e2e-hot-path-agents.sh scripts/fixtures/hot-path-agent-smoke-manifest.schema.json go.mod go.sum ) # Forbidden manifest field names and redaction patterns. The schema rejects these # names via patternProperties->false and closed objects; validate_manifest scans # recursively as a defense-in-depth check. readonly FORBIDDEN_KEY_REGEX='^(prompt|output|token|key|auth|credential|secret|password|api_key|apikey|endpoint|bearer|cookie|session_token)$' readonly REDACTION_PATTERNS=( 'sk-ant-[A-Za-z0-9_-]+' 'pi-fake-PI-SENTINEL-[0-9]+' 'Bearer[ ]?[A-Za-z0-9._-]+' 'RAW-OUTPUT-SENTINEL-[A-Za-z0-9_-]+' 'Summarize the workspace README' 'Author the plan/review pair' 'The seeded file has a defect' 'Perform a long running analysis' ) readonly REDACTION_PATTERN_LABELS=( anthropic_key pi_key bearer_value raw_stdout raw_prompt ) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" SCHEMA_PATH="$SCRIPT_DIR/fixtures/hot-path-agent-smoke-manifest.schema.json" SELF_PATH="$SCRIPT_DIR/e2e-hot-path-agents.sh" # Timeout (seconds) for the timeout-cancel scenario before child-only signaling. readonly CANCEL_TIMEOUT_SEC=1 readonly SHARED_SENTINEL_LIFE_SEC=5 OBSERVATION_WAIT_MSEC=5000 OBSERVATION_CANCEL_WAIT_MSEC=10000 OBSERVATION_QUIET_MSEC=150 log() { printf '[e2e-hot-path-agents] %s\n' "$*" >&2; } die() { log "error: $*"; exit "${EXIT_SOFTWARE}"; } die_usage() { log "usage: $*"; exit "${EXIT_USAGE}"; } die_validation() { log "validation failed: $*"; exit "${EXIT_VALIDATION}"; } require_cmd() { command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" } sha256_str() { printf '%s' "$1" | sha256sum | awk '{printf "sha256:%s", $1}' } sha256_file() { local p="$1" [ -f "$p" ] || die "sha256_file: missing file: $p" sha256sum "$p" | awk '{printf "sha256:%s", $1}' } # Hash sorted relative paths, file sizes, and file bytes. A content-only change # therefore changes the digest without exposing any workspace value. tree_sha256() { local dir="$1" [ -d "$dir" ] || die "tree_sha256: missing dir: $dir" ( cd "$dir" || exit 1 while IFS= read -r -d '' path; do printf 'path:%s\0size:%s\0' "$path" "$(stat -c '%s' -- "$path")" sha256sum -- "$path" | awk '{printf "content:%s\0", $1}' done < <(find . -type f -printf '%P\0' 2>/dev/null | LC_ALL=C sort -z) ) | sha256sum | awk '{printf "sha256:%s", $1}' } git_head() { git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null \ || printf '0000000000000000000000000000000000000000' } git_tree() { git -C "$REPO_ROOT" rev-parse "HEAD:scripts" 2>/dev/null \ || printf '0000000000000000000000000000000000000000' } # Compute a deterministic digest over the current worktree inputs (tracked and # untracked bytes) that back the Hot Path runtime, so external identity binds to # the exact checkout rather than HEAD-only identity. Each per-file `sha256sum` # line carries both content and path, so a content or path change flips the # digest; hashing is batched through xargs so the traversal stays cheap even for # large directories. Never prints file content. compute_worktree_fingerprint() { ( cd "$REPO_ROOT" || exit 1 { local p for p in "${WORKTREE_FINGERPRINT_PATHS[@]}"; do if [ -d "$p" ]; then find "$p" -type f -print0 2>/dev/null elif [ -f "$p" ]; then printf '%s\0' "$p" fi done } | LC_ALL=C sort -z | xargs -0 -r sha256sum ) | sha256sum | awk '{printf "sha256:%s", $1}' } # Memoized worktree fingerprint: the worktree does not change within one run, so # the (potentially large) traversal happens at most once. worktree_fingerprint() { if [ -z "${WORKTREE_FINGERPRINT_CACHE:-}" ]; then WORKTREE_FINGERPRINT_CACHE=$(compute_worktree_fingerprint) fi printf '%s' "$WORKTREE_FINGERPRINT_CACHE" } # Build the exact Claude argv tokens (excluding the binary path). The prompt is # positional; the workspace is supplied via the process working directory and the # base/model are supplied via the environment, so neither becomes an argv token. build_claude_argv() { local prompt="$1" printf '%s\0' "${CLAUDE_FLAGS[@]}" "$prompt" } # Build the exact Pi argv tokens (excluding the binary path). build_pi_argv() { local provider="$1" model="$2" prompt="$3" printf '%s\0' \ "--provider" "$provider" \ "--model" "$model" \ "--mode" "json" \ "--print" \ "--no-session" \ "$prompt" } request_id_for() { local case_id="$1" printf 'rid-%s' "$(sha256_str "$case_id" | sed 's/^sha256://' | cut -c1-8)" } # Deterministic scenario -> preset alias map. Each of the five scenarios binds to # exactly one of the four caller-supplied model aliases so a structurally valid # run must reach the intended IOP preset rather than an arbitrary host default. scenario_model_alias() { case "$1" in direct) printf '%s' "$DIRECT_MODEL" ;; light-pass) printf '%s' "$PASS_MODEL" ;; write-unavailable) printf '%s' "$PASS_MODEL" ;; repair) printf '%s' "$REPAIR_MODEL" ;; timeout-cancel) printf '%s' "$SLOW_MODEL" ;; *) die "unknown scenario: $1" ;; esac } # --------------------------------------------------------------------------- # Input parsing and validation # --------------------------------------------------------------------------- usage() { cat >&2 <<'EOF' usage: e2e-hot-path-agents.sh --self-test e2e-hot-path-agents.sh --preflight-only --claude --pi --runtime-evidence --base-url --direct-model --pass-model --repair-model --slow-model --edge-bin --edge-config --pi-config-dir --pi-provider --observation-file --workspace-root --output --claude-secret-env --pi-secret-env [--fixture ] e2e-hot-path-agents.sh --run (same inputs as --preflight-only) EOF } parse_run_inputs() { CLAUDE_BIN="" PI_BIN="" RUNTIME_EVIDENCE="" FIXTURE_PATH="$SCHEMA_PATH" BASE_URL="" DIRECT_MODEL="" PASS_MODEL="" REPAIR_MODEL="" SLOW_MODEL="" EDGE_BIN="" EDGE_CONFIG="" PI_CONFIG_DIR="" PI_PROVIDER="" OBSERVATION_FILE="" WORKSPACE_ROOT="" OUTPUT_PATH="" CLAUDE_SECRET_ENV="" PI_SECRET_ENV="" while [ "$#" -gt 0 ]; do case "$1" in --claude) CLAUDE_BIN="${2:-}"; shift 2 ;; --pi) PI_BIN="${2:-}"; shift 2 ;; --runtime-evidence) RUNTIME_EVIDENCE="${2:-}"; shift 2 ;; --fixture) FIXTURE_PATH="${2:-}"; shift 2 ;; --base-url) BASE_URL="${2:-}"; shift 2 ;; --direct-model) DIRECT_MODEL="${2:-}"; shift 2 ;; --pass-model) PASS_MODEL="${2:-}"; shift 2 ;; --repair-model) REPAIR_MODEL="${2:-}"; shift 2 ;; --slow-model) SLOW_MODEL="${2:-}"; shift 2 ;; --edge-bin) EDGE_BIN="${2:-}"; shift 2 ;; --edge-config) EDGE_CONFIG="${2:-}"; shift 2 ;; --pi-config-dir) PI_CONFIG_DIR="${2:-}"; shift 2 ;; --pi-provider) PI_PROVIDER="${2:-}"; shift 2 ;; --observation-file) OBSERVATION_FILE="${2:-}"; shift 2 ;; --workspace-root) WORKSPACE_ROOT="${2:-}"; shift 2 ;; --output) OUTPUT_PATH="${2:-}"; shift 2 ;; --claude-secret-env) CLAUDE_SECRET_ENV="${2:-}"; shift 2 ;; --pi-secret-env) PI_SECRET_ENV="${2:-}"; shift 2 ;; *) die_usage "unknown option: $1" ;; esac done } validate_inputs_presence() { [ -n "$CLAUDE_BIN" ] || die_validation "missing --claude binary" [ -n "$PI_BIN" ] || die_validation "missing --pi binary" [ -x "$CLAUDE_BIN" ] || die_validation "claude binary not executable" [ -x "$PI_BIN" ] || die_validation "pi binary not executable" [ -n "$RUNTIME_EVIDENCE" ] || die_validation "missing --runtime-evidence" [ -f "$RUNTIME_EVIDENCE" ] || die_validation "runtime-evidence file absent" [ -n "$FIXTURE_PATH" ] || die_validation "missing --fixture" [ -f "$FIXTURE_PATH" ] || die_validation "fixture file absent" [ -n "$BASE_URL" ] || die_validation "missing --base-url" [ -n "$DIRECT_MODEL" ] || die_validation "missing --direct-model alias" [ -n "$PASS_MODEL" ] || die_validation "missing --pass-model alias" [ -n "$REPAIR_MODEL" ] || die_validation "missing --repair-model alias" [ -n "$SLOW_MODEL" ] || die_validation "missing --slow-model alias" [ -n "$EDGE_BIN" ] || die_validation "missing --edge-bin" [ -x "$EDGE_BIN" ] || die_validation "edge binary not executable" [ -n "$EDGE_CONFIG" ] || die_validation "missing --edge-config" [ -f "$EDGE_CONFIG" ] || die_validation "edge-config file absent" [ -n "$PI_CONFIG_DIR" ] || die_validation "missing --pi-config-dir" [ -d "$PI_CONFIG_DIR" ] || die_validation "pi-config-dir absent" [ -n "$PI_PROVIDER" ] || die_validation "missing --pi-provider" [ -n "$OBSERVATION_FILE" ] || die_validation "missing --observation-file" [ -f "$OBSERVATION_FILE" ] || die_validation "observation-file absent" [ -n "$WORKSPACE_ROOT" ] || die_validation "missing --workspace-root" [ -d "$WORKSPACE_ROOT" ] || die_validation "workspace-root absent" [ -n "$OUTPUT_PATH" ] || die_validation "missing --output" [ -n "$CLAUDE_SECRET_ENV" ] || die_validation "missing --claude-secret-env" [ -n "$PI_SECRET_ENV" ] || die_validation "missing --pi-secret-env" # Presence-only secret check: the named env vars must be set and non-empty. # Values are never read or printed. [ -n "${!CLAUDE_SECRET_ENV:-}" ] || die_validation "claude secret env not present" [ -n "${!PI_SECRET_ENV:-}" ] || die_validation "pi secret env not present" return 0 } # Compare a caller-supplied evidence field to an actual computed value without # ever echoing either value (only the field name appears on mismatch). assert_digest_matches() { local actual="$1" supplied_file="$2" field="$3" local supplied supplied=$(jq -r --arg f "$field" '.[$f] // empty' "$supplied_file" 2>/dev/null) \ || die_validation "$field: evidence file is not valid JSON" [ -n "$supplied" ] || die_validation "$field: missing from evidence" [ "$supplied" = "$actual" ] || die_validation "$field: identity mismatch" } # Validate current source/worktree identity against the runtime evidence. Sets # SOURCE_HEAD/SOURCE_TREE for the manifest. validate_worktree_fingerprint() { local actual_script actual_schema actual_fp actual_script=$(sha256_file "$SELF_PATH") actual_schema=$(sha256_file "$SCHEMA_PATH") assert_digest_matches "$actual_script" "$RUNTIME_EVIDENCE" "script_sha256" assert_digest_matches "$actual_schema" "$RUNTIME_EVIDENCE" "schema_sha256" SOURCE_HEAD=$(git_head) SOURCE_TREE=$(git_tree) assert_digest_matches "$SOURCE_HEAD" "$RUNTIME_EVIDENCE" "head" assert_digest_matches "$SOURCE_TREE" "$RUNTIME_EVIDENCE" "source_tree" actual_fp=$(worktree_fingerprint) assert_digest_matches "$actual_fp" "$RUNTIME_EVIDENCE" "worktree_fingerprint" } # Validate the selected Edge binary/config, Pi config dir, CLI binaries and the # fixture against the runtime evidence. Sets manifest digest globals. validate_edge_binary_config_fixture_identity() { local actual_claude actual_pi actual_edge actual_edge_cfg actual_pi_cfg actual_fixture actual_claude=$(sha256_file "$CLAUDE_BIN") actual_pi=$(sha256_file "$PI_BIN") actual_edge=$(sha256_file "$EDGE_BIN") actual_edge_cfg=$(sha256_file "$EDGE_CONFIG") actual_pi_cfg=$(tree_sha256 "$PI_CONFIG_DIR") actual_fixture=$(sha256_file "$FIXTURE_PATH") assert_digest_matches "$actual_claude" "$RUNTIME_EVIDENCE" "claude_binary_sha256" assert_digest_matches "$actual_pi" "$RUNTIME_EVIDENCE" "pi_binary_sha256" assert_digest_matches "$actual_edge" "$RUNTIME_EVIDENCE" "edge_binary_sha256" assert_digest_matches "$actual_edge_cfg" "$RUNTIME_EVIDENCE" "edge_config_sha256" assert_digest_matches "$actual_pi_cfg" "$RUNTIME_EVIDENCE" "pi_config_sha256" assert_digest_matches "$actual_fixture" "$RUNTIME_EVIDENCE" "fixture_sha256" RUNTIME_SHA256=$(sha256_file "$RUNTIME_EVIDENCE") FIXTURE_SHA256="$actual_fixture" CLAUDE_BIN_SHA256="$actual_claude" PI_BIN_SHA256="$actual_pi" } # Validate the base/profile identity and the four scenario preset aliases against # the runtime evidence by digest only. Endpoint and model values are never # printed or serialized. validate_runner_and_profile_identity() { assert_digest_matches "$(sha256_str "$BASE_URL")" "$RUNTIME_EVIDENCE" "base_url_sha256" assert_digest_matches "$(sha256_str "$PI_PROVIDER")" "$RUNTIME_EVIDENCE" "pi_provider_sha256" assert_digest_matches "$(sha256_str "$DIRECT_MODEL")" "$RUNTIME_EVIDENCE" "direct_model_sha256" assert_digest_matches "$(sha256_str "$PASS_MODEL")" "$RUNTIME_EVIDENCE" "pass_model_sha256" assert_digest_matches "$(sha256_str "$REPAIR_MODEL")" "$RUNTIME_EVIDENCE" "repair_model_sha256" assert_digest_matches "$(sha256_str "$SLOW_MODEL")" "$RUNTIME_EVIDENCE" "slow_model_sha256" } # The observation log must be a readable regular file (a live Edge log holding # JSON hot_path_observation records). Per-case freshness is enforced during the # run, not here. validate_observation_log_preflight() { [ -f "$OBSERVATION_FILE" ] || die_validation "observation-file is not a regular file" [ -r "$OBSERVATION_FILE" ] || die_validation "observation-file not readable" log "observation log preflight ok" } # --------------------------------------------------------------------------- # Scenario fixtures and agent invocation # --------------------------------------------------------------------------- scenario_prompt() { case "$1" in direct) printf 'Summarize the workspace README in one short line.' ;; light-pass) printf 'Author the plan/review pair and complete the task.' ;; repair) printf 'The seeded file has a defect; author plan/review, fix and verify.' ;; write-unavailable) printf 'Author the plan/review pair under the job directory.' ;; timeout-cancel) printf 'Perform a long running analysis of the workspace.' ;; *) die "unknown scenario: $1" ;; esac } # Validate one production Hot Path lifecycle and collapse retry attempts into the # manifest's one-row-per-stage projection. Return 2 while the lifecycle is still # open and 1 for a closed contradiction or malformed production record. reduce_observation_fragment() { local scenario="$1" frag="$2" projected projected=$(jq -c -s ' . as $all | if any($all[]; type == "object" and (.msg // "") != "hot_path_observation" and (has("hot_path_event_class") or has("hot_path_request_id"))) then error("foreign hot path observation lookalike") else [ $all[] | select(type == "object" and .msg == "hot_path_observation") | {raw_rid:(.hot_path_request_id // ""), ec:(.hot_path_event_class // ""), sk:(.hot_path_stage_kind // ""), attempt:(.hot_path_attempt_bucket // ""), disposition:(.hot_path_disposition // ""), reason:(.hot_path_reason // ""), cleanup:(.hot_path_cleanup_outcome // ""), orphan:(.hot_path_orphan_outcome // "")} ] end ' "$frag" 2>/dev/null) || return 1 local record_count rid_count record_count=$(jq 'length' <<<"$projected") || return 1 [ "$record_count" -gt 0 ] || return 2 jq -e ' def oneof($xs): . as $v | any($xs[]; . == $v); all(.[]; (.raw_rid | type == "string" and length > 0) and (.ec | oneof(["dispatch","stage","light","terminal","cleanup","orphan"])) and (.sk | oneof(["","selector","local","review","cleanup"])) and (.attempt | oneof(["","first","retry"])) and (.disposition | oneof(["","success","tool_turn","length","provider_error","validation_error","timeout","caller_cancel"])) and (.reason | oneof(["","mode_disabled","artifact_required","invalid_input","provider_error","timeout","caller_cancel"])) and (.cleanup | oneof(["","success","primary_error","ttl_expired"])) and (.orphan | oneof(["","ttl_expired","cleanup_failed"])) and (if .ec == "dispatch" then .sk == "" and .attempt == "" and .disposition == "" and .cleanup == "" and .orphan == "" elif .ec == "stage" then (.sk | IN("local","review")) and (.attempt | IN("first","retry")) and (.disposition != "") and .reason == "" and .cleanup == "" and .orphan == "" elif .ec == "light" then (.sk | IN("review","cleanup")) and (.attempt | IN("first","retry")) and .disposition == "" and .reason == "" and .cleanup == "" and .orphan == "" elif .ec == "terminal" then .sk == "" and .attempt == "" and .disposition != "" and .reason == "" and .cleanup == "" and .orphan == "" elif .ec == "cleanup" then .sk == "" and .attempt == "" and .disposition == "" and .reason == "" and .cleanup != "" and .orphan == "" else .sk == "" and .attempt == "" and .disposition == "" and .reason == "" and .cleanup == "" and .orphan != "" end) ) ' <<<"$projected" >/dev/null 2>&1 || return 1 rid_count=$(jq '[.[].raw_rid] | unique | length' <<<"$projected") || return 1 [ "$rid_count" -eq 1 ] || return 1 local closure_count if [ "$scenario" = timeout-cancel ]; then closure_count=$(jq '[.[] | select(.ec == "stage" and .sk == "local" and (.disposition | IN("caller_cancel","timeout")))] | length' <<<"$projected") || return 1 elif [ "$scenario" = write-unavailable ]; then closure_count=$(jq '[.[] | select(.ec == "dispatch" and .reason != "")] | length' <<<"$projected") || return 1 else closure_count=$(jq '[.[] | select(.ec == "terminal")] | length' <<<"$projected") || return 1 fi [ "$closure_count" -gt 0 ] || return 2 # The production lifecycle is closed by a terminal for admitted direct/light # cases, by the bounded rejection reason for failed admission, and by the # immediate local caller-cancel stage for harness-owned child cancellation. # Stage attempts may repeat, but only a terminal success can close each # successful stage. jq -e --arg scenario "$scenario" ' def stage_rows($kind): [to_entries[] | select(.value.ec == "stage" and .value.sk == $kind)]; def light_rows($kind): [to_entries[] | select(.value.ec == "light" and .value.sk == $kind)]; def attempts_close($rows): ($rows | length) > 0 and $rows[0].value.attempt == "first" and all($rows[1:][]; .value.attempt == "retry") and all($rows[0:-1][]; .value.disposition == "tool_turn") and $rows[-1].value.disposition == "success"; . as $p | if $scenario == "direct" then ($p | length) == 2 and $p[0].ec == "dispatch" and $p[0].reason == "" and $p[1].ec == "terminal" and $p[1].disposition == "success" elif $scenario == "write-unavailable" then ($p | length) == 1 and $p[0].ec == "dispatch" and $p[0].reason != "" elif ($scenario == "light-pass" or $scenario == "repair") then stage_rows("local") as $local | stage_rows("review") as $review | light_rows("review") as $review_transition | light_rows("cleanup") as $cleanup_transition | [to_entries[] | select(.value.ec == "cleanup")] as $cleanup | [to_entries[] | select(.value.ec == "terminal")] as $terminal | [to_entries[] | select(.value.ec == "dispatch")] as $dispatch | [to_entries[] | select(.value.ec == "orphan")] as $orphan | ($dispatch | length) == 1 and $dispatch[0].key == 0 and $dispatch[0].value.reason == "" and attempts_close($local) and attempts_close($review) and ($review_transition | length) == (if $scenario == "repair" then 2 else 1 end) and $review_transition[0].value.attempt == "first" and all($review_transition[1:][]; .value.attempt == "retry") and ($cleanup_transition | length) == 1 and $cleanup_transition[0].value.attempt == "first" and ($cleanup | length) == 1 and $cleanup[0].value.cleanup == "success" and ($terminal | length) == 1 and $terminal[0].value.disposition == "success" and ($orphan | length) == 0 and $local[0].key == 1 and $local[-1].key < $review_transition[0].key and $review_transition[0].key < $review[0].key and $review[-1].key < $cleanup_transition[0].key and $cleanup_transition[0].key + 1 == $cleanup[0].key and $cleanup[0].key + 1 == $terminal[0].key and $terminal[0].key + 1 == ($p | length) and ($p | length) == (1 + ($local|length) + ($review|length) + ($review_transition|length) + 1 + 1 + 1) else stage_rows("local") as $local | [to_entries[] | select(.value.ec == "dispatch")] as $dispatch | [to_entries[] | select(.value.ec == "orphan")] as $orphan | [to_entries[] | select(.value.ec == "terminal" or .value.ec == "cleanup" or .value.ec == "light" or (.value.ec == "stage" and .value.sk != "local"))] as $foreign | ($dispatch | length) == 1 and $dispatch[0].key == 0 and $dispatch[0].value.reason == "" and ($local | length) > 0 and $local[0].key == 1 and $local[0].value.attempt == "first" and all($local[1:][]; .value.attempt == "retry") and all($local[0:-1][]; .value.disposition == "tool_turn") and ($local[-1].value.disposition | IN("caller_cancel","timeout")) and ($orphan | length) == 0 and $local[-1].key + 1 == ($p | length) and ($foreign | length) == 0 and ($p | length) == (1 + ($local|length)) end ' <<<"$projected" >/dev/null 2>&1 || return 1 local raw_rid proj_rid raw_rid=$(jq -r '.[0].raw_rid' <<<"$projected") || return 1 proj_rid="rid-$(sha256_str "$raw_rid" | sed 's/^sha256://' | cut -c1-8)" jq -c --arg rid "$proj_rid" --arg scenario "$scenario" ' if $scenario == "direct" then [{request_id:$rid,stage:"selector",outcome:"observed"}] elif $scenario == "write-unavailable" then [{request_id:$rid,stage:"selector",outcome:"failed"}] elif $scenario == "timeout-cancel" then [{request_id:$rid,stage:"selector",outcome:"observed"}, {request_id:$rid,stage:"local",outcome:"observed"}] else [{request_id:$rid,stage:"selector",outcome:"observed"}, {request_id:$rid,stage:"local",outcome:"observed"}, {request_id:$rid,stage:"review",outcome:"observed"}, {request_id:$rid,stage:"cleanup",outcome:"observed"}] end ' <<<"$projected" } # Read the observation records appended by the selected runtime to the live log # after the case started. Poll until a scenario-specific closure is stable for a # short quiet interval, bounded by the configured lifecycle deadline. capture_appended_observation() { local case_id="$1" scenario="$2" offset_before="$3" inode_before="$4" local f="$OBSERVATION_FILE" [ -f "$f" ] || return 1 local frag="$RAW_CAPTURE_DIR/obs-appended-${case_id}" local wait_msec="$OBSERVATION_WAIT_MSEC" [ "$scenario" = timeout-cancel ] && wait_msec="$OBSERVATION_CANCEL_WAIT_MSEC" local start_ms now_ms deadline_ms cur_inode cur_size last_closed_size=-1 closed_since=0 local candidate rc start_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || printf '0') deadline_ms=$((start_ms + wait_msec)) while :; do [ -f "$f" ] || return 1 cur_inode=$(stat -c '%i' "$f" 2>/dev/null || printf '0') cur_size=$(stat -c '%s' "$f" 2>/dev/null || printf '0') [ "$cur_inode" = "$inode_before" ] || return 1 [ "$cur_size" -ge "$offset_before" ] || return 1 tail -c "+$((offset_before + 1))" "$f" > "$frag" 2>/dev/null || return 1 if candidate=$(reduce_observation_fragment "$scenario" "$frag"); then rc=0 else rc=$? fi [ "$rc" -ne 1 ] || return 1 now_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || printf '0') if [ "$rc" -eq 0 ]; then if [ "$cur_size" -ne "$last_closed_size" ]; then last_closed_size="$cur_size" closed_since="$now_ms" elif [ $((now_ms - closed_since)) -ge "$OBSERVATION_QUIET_MSEC" ]; then printf '%s' "$candidate" return 0 fi else last_closed_size=-1 closed_since=0 fi [ "$now_ms" -lt "$deadline_ms" ] || return 1 sleep 0.05 done } workspace_snapshot() { local ws="$1" local artifacts=false if [ -e "$ws/.iop/job" ] && [ -n "$(find "$ws/.iop/job" -mindepth 1 -print -quit 2>/dev/null)" ]; then artifacts=true fi local writable=false mode mode=$(stat -c '%A' "$ws") if [[ "${mode:2:1}${mode:5:1}${mode:8:1}" == *w* ]]; then writable=true; fi printf '{"artifacts_present":%s,"writable":%s,"tree_sha256":"%s"}' \ "$artifacts" "$writable" "$(tree_sha256 "$ws")" } # Parse captured agent stdout (JSONL) into visible_event summaries. The agent # field selects the native shape. Raw content is never emitted; only safe kinds # and short labels are recorded. A single jq pass parses the whole stream so the # per-event subprocess pipeline cost (catastrophic on slow filesystems) is # avoided and the visible_event index stays deterministically sequential. parse_visible_events() { local agent="$1" out_file="$2" child_status="${3:-0}" local triggered="${4:-false}" target="${5:-none}" events events=$(jq -c -s --arg agent "$agent" ' def tool_detail($name; $args): ($name // "" | ascii_downcase) as $n | ($args // {} | tojson | ascii_downcase) as $a | if ($n | test("cleanup|delete|remove")) or (($a | test("\\.iop/job")) and ($a | test("rm |delete|remove"))) then "workspace_cleanup" elif ($n | test("repair")) or ($a | test("seeded\\.txt|repair")) then "repair_write" elif ($n | test("review")) or ($a | test("review\\.md")) then "review_write" elif ($n | test("write|plan")) or ($a | test("plan\\.md|\\.iop/job")) then "workspace_write" else "tool_call" end; if $agent == "claude" then [ .[] | if .type == "system" then {kind:"system_init", detail:"init"} elif .type == "assistant" then (.message.content // [])[] | if .type == "tool_use" then {kind:"tool_use", detail:tool_detail(.name; .input)} else {kind:"assistant_text", detail:"text"} end elif .type == "user" then (.message.content // [])[] | if .type == "tool_result" then if (.is_error // false) then {kind:"tool_result", detail:"error"} else {kind:"tool_result", detail:"ok"} end else {kind:"partial", detail:"event"} end elif .type == "result" then if .subtype == "success" then {kind:"terminal_success", detail:"success"} elif (.subtype | IN("cancelled","canceled","interrupted")) then {kind:"terminal_cancelled", detail:"cancelled"} else {kind:"terminal_error", detail:"provider_error"} end else empty end ] else reduce .[] as $e ({visible:[], final_assistant:null, agent_end_count:0, invalid:false}; if $e.type == "agent_start" then .visible += [{kind:"system_init",detail:"init"}] elif $e.type == "message_update" and $e.message.role == "assistant" then .visible += [{kind:"partial",detail:"delta"}] elif $e.type == "message_end" and $e.message.role == "assistant" then .final_assistant = $e.message | if any($e.message.content[]?; .type == "text" or .type == "thinking") then .visible += [{kind:"assistant_text",detail:"text"}] else . end elif $e.type == "tool_execution_start" then .visible += [{kind:"tool_use",detail:tool_detail($e.toolName;$e.args)}] elif $e.type == "tool_execution_end" then .visible += [{kind:"tool_result",detail:(if ($e.isError // false) then "error" else "ok" end)}] elif $e.type == "agent_end" then .agent_end_count += 1 | (([$e.messages[]? | select(.role == "assistant")] | last) // .final_assistant) as $final | if $final == null then .invalid = true elif $final.stopReason == "stop" then .visible += [{kind:"terminal_success",detail:"success"}] elif ($final.stopReason | IN("error","aborted","length","toolUse")) then .visible += [{kind:"terminal_error",detail:"provider_error"}] else .invalid = true end else . end) | if .invalid or .agent_end_count > 1 then error("invalid Pi lifecycle") else .visible end end | to_entries | map({index:.key, kind:.value.kind, detail:.value.detail}) ' "$out_file" 2>/dev/null) || return 1 # Pi 0.81.1 disposes and exits 143 on SIGTERM without an AgentSessionEvent # terminal. Only the harness-owned child-only signal may close that exact # process state as cancellation, and never over a contradictory terminal. if [ "$agent" = pi ] && [ "$child_status" -eq 143 ] \ && [ "$triggered" = true ] && [ "$target" = child_only ]; then local terminal_count next_index terminal_count=$(jq '[.[] | select(.kind | startswith("terminal_"))] | length' <<<"$events") || return 1 if [ "$terminal_count" -eq 0 ]; then next_index=$(jq 'length' <<<"$events") || return 1 events=$(jq -c --argjson i "$next_index" \ '. + [{index:$i,kind:"terminal_cancelled",detail:"cancelled"}]' <<<"$events") || return 1 fi fi printf '%s' "$events" } # Derive the public result only from correlated process, protocol, observation, # cancellation, and workspace facts. Scenario names select invariants; they are # never copied into outcome/terminal/cleanup without these checks succeeding. derive_case_result() { local agent="$1" scenario="$2" child_status="$3" triggered="$4" target="$5" local sentinel_survived="$6" visible_events="$7" observation="$8" local snapshot_before="$9" snapshot_after="${10}" local terminal_kind terminal_count outcome terminal cleanup terminal_count=$(jq '[.[] | select(.kind | startswith("terminal_"))] | length' <<<"$visible_events") [ "$terminal_count" -eq 1 ] || return 1 jq -e 'length > 0 and (.[-1].kind | startswith("terminal_"))' \ <<<"$visible_events" >/dev/null || return 1 terminal_kind=$(jq -r '.[-1].kind' <<<"$visible_events") case "$terminal_kind" in terminal_success) [ "$child_status" -eq 0 ] && [ "$triggered" = false ] && [ "$target" = none ] || return 1 outcome=completed; terminal=success ;; terminal_error) case "$agent" in pi) [ "$child_status" -eq 0 ] ;; claude) [ "$child_status" -ne 0 ] ;; *) return 1 ;; esac [ "$triggered" = false ] && [ "$target" = none ] || return 1 outcome=error; terminal=provider_error ;; terminal_cancelled) [ "$child_status" -ne 0 ] && [ "$triggered" = true ] \ && [ "$target" = child_only ] && [ "$sentinel_survived" = true ] || return 1 outcome=cancelled; terminal=cancelled ;; *) return 1 ;; esac [ "$sentinel_survived" = true ] || return 1 # A terminal-only stream is not evidence that the agent exposed the Hot Path # work. Require the scenario's visible tool progression in causal order. case "$scenario" in direct) jq -e 'any(.[]; .kind == "assistant_text" or .kind == "partial")' \ <<<"$visible_events" >/dev/null || return 1 ;; light-pass) jq -e ' [.[] | select(.kind == "tool_use") | .detail] as $t | ($t | index("workspace_write")) as $write | ($t | index("review_write")) as $review | ($t | index("workspace_cleanup")) as $cleanup | $write != null and $review != null and $cleanup != null and $write < $review and $review < $cleanup ' <<<"$visible_events" >/dev/null || return 1 ;; repair) jq -e ' [.[] | select(.kind == "tool_use") | .detail] as $t | ($t | index("workspace_write")) as $write | ($t | index("review_write")) as $review | ($t | index("repair_write")) as $repair | ($t | index("workspace_cleanup")) as $cleanup | $write != null and $review != null and $repair != null and $cleanup != null and $write < $review and $review < $repair and $repair < $cleanup ' <<<"$visible_events" >/dev/null || return 1 ;; write-unavailable) jq -e ' any(.[]; .kind == "tool_use" and .detail == "workspace_write") and any(.[]; .kind == "tool_result" and .detail == "error") ' <<<"$visible_events" >/dev/null || return 1 ;; timeout-cancel) jq -e 'any(.[]; .kind == "tool_use" and .detail == "workspace_write")' \ <<<"$visible_events" >/dev/null || return 1 ;; esac if [ "$terminal" = cancelled ] \ && jq -e '.artifacts_present == true' <<<"$snapshot_after" >/dev/null; then cleanup=orphan elif jq -e 'any(.[]; .stage == "cleanup" and .outcome == "observed")' \ <<<"$observation" >/dev/null \ && jq -e '.artifacts_present == false' <<<"$snapshot_after" >/dev/null; then cleanup=removed else cleanup=none fi case "$scenario" in direct) [ "$outcome:$terminal:$cleanup" = "completed:success:none" ] || return 1 jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" ' ($b.artifacts_present == false) and ($a.artifacts_present == false) and ($b.writable == true) and ($a.writable == true) and ($b.tree_sha256 == $a.tree_sha256) ' -n >/dev/null || return 1 ;; light-pass|repair) [ "$outcome:$terminal:$cleanup" = "completed:success:removed" ] || return 1 jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" ' ($b.artifacts_present == false) and ($a.artifacts_present == false) and ($b.writable == true) and ($a.writable == true) and ($b.tree_sha256 != $a.tree_sha256) ' -n >/dev/null || return 1 ;; write-unavailable) [ "$outcome:$terminal:$cleanup" = "error:provider_error:none" ] || return 1 jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" ' ($b.artifacts_present == false) and ($a.artifacts_present == false) and ($b.writable == false) and ($a.writable == false) and ($b.tree_sha256 == $a.tree_sha256) ' -n >/dev/null || return 1 ;; timeout-cancel) [ "$outcome:$terminal:$cleanup" = "cancelled:cancelled:orphan" ] || return 1 jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" ' ($b.artifacts_present == false) and ($a.artifacts_present == true) and ($b.writable == true) and ($a.writable == true) and ($b.tree_sha256 != $a.tree_sha256) ' -n >/dev/null || return 1 ;; *) return 1 ;; esac printf '%s:%s:%s' "$outcome" "$terminal" "$cleanup" } # Run a single matrix case. Produces a case evidence JSON object on stdout. run_case() { local agent="$1" scenario="$2" case_id="$agent:$scenario" request_id request_id=$(request_id_for "$case_id") local ws="$WORKSPACE_ROOT/$case_id" rm -rf "$ws" mkdir -p "$ws" if [ "$scenario" = repair ]; then printf 'defect marker\n' > "$ws/seeded.txt"; fi if [ "$scenario" = write-unavailable ]; then chmod a-w "$ws" || return 1 fi local snapshot_before snapshot_before=$(workspace_snapshot "$ws") local prompt model provider agent_bin prompt=$(scenario_prompt "$scenario") model=$(scenario_model_alias "$scenario") if [ "$agent" = claude ]; then provider="claude"; agent_bin="$CLAUDE_BIN" else provider="$PI_PROVIDER"; agent_bin="$PI_BIN" fi local argv_file="$RAW_CAPTURE_DIR/argv-${case_id}.expected" local recorded_file="$RAW_CAPTURE_DIR/argv-${case_id}.recorded" local out_file="$RAW_CAPTURE_DIR/out-${case_id}.jsonl" local err_file="$RAW_CAPTURE_DIR/err-${case_id}.log" if [ "$agent" = claude ]; then build_claude_argv "$prompt" > "$argv_file" else build_pi_argv "$provider" "$model" "$prompt" > "$argv_file"; fi local argv_hash argv_hash=$(sha256_file "$argv_file") : > "$out_file"; : > "$err_file" local -a argv_arr=() local tok while IFS= read -r -d '' tok; do argv_arr+=("$tok"); done < "$argv_file" # Snapshot the observation-log identity and byte offset immediately before # invocation so only records appended by this case are consumed afterward. local obs_offset_before obs_inode_before obs_offset_before=$(stat -c '%s' "$OBSERVATION_FILE" 2>/dev/null || printf '0') obs_inode_before=$(stat -c '%i' "$OBSERVATION_FILE" 2>/dev/null || printf '0') local sentinel_pid child_pid sleep "$SHARED_SENTINEL_LIFE_SEC" >/dev/null 2>&1 & sentinel_pid=$! local triggered=false target=none start_ms end_ms child_status=0 start_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || echo 0) ( cd "$ws" || exit 1 ANTHROPIC_BASE_URL="$BASE_URL" \ ANTHROPIC_MODEL="$model" \ PI_CODING_AGENT_DIR="$PI_CONFIG_DIR" \ IOP_HOT_PATH_FAKE_AGENT="$agent" \ IOP_HOT_PATH_FAKE_SCENARIO="$scenario" \ IOP_HOT_PATH_FAKE_REQUEST_ID="$request_id" \ IOP_HOT_PATH_FAKE_WORKSPACE="$ws" \ IOP_HOT_PATH_FAKE_RECORD="$recorded_file" \ IOP_HOT_PATH_FAKE_INVOCATION_MARKER="$INVOCATION_MARKER" \ IOP_HOT_PATH_FAKE_OBSERVATION_FILE="$OBSERVATION_FILE" \ exec "$agent_bin" "${argv_arr[@]}" ) >"$out_file" 2>"$err_file" & child_pid=$! if [ "$scenario" = timeout-cancel ]; then sleep "$CANCEL_TIMEOUT_SEC" if kill -0 "$child_pid" 2>/dev/null; then kill -TERM "$child_pid" 2>/dev/null || true triggered=true target=child_only fi fi wait "$child_pid" 2>/dev/null || child_status=$? end_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || echo 0) local duration_ms=$(( end_ms - start_ms )) [ "$duration_ms" -lt 0 ] && duration_ms=0 local sentinel_survived=false if kill -0 "$sentinel_pid" 2>/dev/null; then sentinel_survived=true; fi kill "$sentinel_pid" 2>/dev/null || true wait "$sentinel_pid" 2>/dev/null || true local snapshot_after snapshot_after=$(workspace_snapshot "$ws") if [ "$scenario" = write-unavailable ]; then chmod u+w "$ws" 2>/dev/null || true; fi if [ "${REQUIRE_RECORDED_ARGV:-false}" = true ]; then cmp -s "$argv_file" "$recorded_file" || return 1 fi local visible_events observation derived outcome terminal cleanup rest visible_events=$(parse_visible_events "$agent" "$out_file" "$child_status" "$triggered" "$target") || return 1 observation=$(capture_appended_observation "$case_id" "$scenario" \ "$obs_offset_before" "$obs_inode_before") || return 1 derived=$(derive_case_result "$agent" "$scenario" "$child_status" "$triggered" "$target" \ "$sentinel_survived" "$visible_events" "$observation" "$snapshot_before" "$snapshot_after") \ || return 1 outcome="${derived%%:*}"; rest="${derived#*:}" terminal="${rest%%:*}"; cleanup="${rest##*:}" jq -n \ --arg id "$case_id" --arg agent "$agent" --arg scenario "$scenario" \ --arg argv_hash "$argv_hash" --arg outcome "$outcome" --arg terminal "$terminal" \ --arg cleanup "$cleanup" --argjson process_exit "$child_status" \ --argjson visible_events "$visible_events" --argjson observation "$observation" \ --argjson ws_before "$snapshot_before" --argjson ws_after "$snapshot_after" \ --argjson triggered "$triggered" --arg target "$target" \ --argjson sentinel_survived "$sentinel_survived" --argjson duration_ms "$duration_ms" ' { id:$id, agent:$agent, scenario:$scenario, argv_hash:$argv_hash, process_exit:$process_exit, outcome:$outcome, terminal:$terminal, cleanup:$cleanup, visible_events:$visible_events, observation:$observation, workspace_before:$ws_before, workspace_after:$ws_after, cancellation:{triggered:$triggered,target:$target,sentinel_survived:$sentinel_survived}, duration_ms:$duration_ms }' } run_matrix() { CASE_RESULTS=() for agent in "${AGENTS[@]}"; do for scenario in "${SCENARIOS[@]}"; do local case_json if ! case_json=$(run_case "$agent" "$scenario"); then log "case evidence rejected: $agent:$scenario" return 1 fi CASE_RESULTS+=("$case_json") done done } # --------------------------------------------------------------------------- # Manifest assembly, validation, redaction, atomic output # --------------------------------------------------------------------------- build_manifest() { local cases_array='[' local first=1 for c in "${CASE_RESULTS[@]}"; do [ "$first" -eq 1 ] || cases_array+=',' cases_array+="$c" first=0 done cases_array+=']' local claude_secret_present=false pi_secret_present=false [ -n "${!CLAUDE_SECRET_ENV:-}" ] && claude_secret_present=true [ -n "${!PI_SECRET_ENV:-}" ] && pi_secret_present=true local obs_hash ws_root_hash run_id # Digest the closed, projected observation evidence actually consumed by the # matrix (never the live log file bytes). obs_hash=$(printf '%s' "$cases_array" | jq -cS '[.[].observation]' \ | sha256sum | awk '{printf "sha256:%s", $1}') ws_root_hash=$(sha256_str "$(cd "$WORKSPACE_ROOT" && pwd)") run_id=$(sha256_str "${SOURCE_HEAD}-${SOURCE_TREE}-${RUNTIME_SHA256}-${cases_array}") local redaction_patterns_json sentinels_seeded_count redaction_patterns_json=$(printf '%s\n' "${REDACTION_PATTERN_LABELS[@]}" | jq -R . | jq -sc .) sentinels_seeded_count="${SENTINELS_SEEDED:-0}" jq -n \ --arg schema_version "$SCHEMA_VERSION" \ --arg run_id "$run_id" \ --arg head "$SOURCE_HEAD" \ --arg source_tree "$SOURCE_TREE" \ --arg script_sha256 "$(sha256_file "$SELF_PATH")" \ --arg schema_sha256 "$(sha256_file "$SCHEMA_PATH")" \ --arg runtime_sha256 "$RUNTIME_SHA256" \ --arg fixture_sha256 "$FIXTURE_SHA256" \ --arg observation_sha256 "$obs_hash" \ --arg workspace_root_hash "$ws_root_hash" \ --arg claude_binary_sha256 "$CLAUDE_BIN_SHA256" \ --arg pi_binary_sha256 "$PI_BIN_SHA256" \ --argjson claude_secret_present "$claude_secret_present" \ --argjson pi_secret_present "$pi_secret_present" \ --argjson cases "$cases_array" \ --argjson redaction_patterns "$redaction_patterns_json" \ --argjson sentinels_seeded "$sentinels_seeded_count" \ '{ schema_version: $schema_version, run_id: $run_id, source: { head: $head, source_tree: $source_tree, script_sha256: $script_sha256, schema_sha256: $schema_sha256 }, runtime: { runtime_sha256: $runtime_sha256, fixture_sha256: $fixture_sha256, observation_sha256: $observation_sha256, workspace_root_hash: $workspace_root_hash }, runner: { claude_binary_sha256: $claude_binary_sha256, pi_binary_sha256: $pi_binary_sha256, claude_secret_present: $claude_secret_present, pi_secret_present: $pi_secret_present, claude_flags: ["--print","--output-format","stream-json","--include-partial-messages","--no-session-persistence","--bare"], pi_flags: ["--provider","--model","--mode","json","--print","--no-session"] }, cases: $cases, redaction: { patterns: $redaction_patterns, sentinels_seeded: $sentinels_seeded, matches: 0 } }' } # Recursive forbidden-key scan over a JSON document. Emits the final key/index # of every jq path and flags any forbidden field name anywhere in the document # (defense-in-depth alongside the closed additionalProperties:false schema). scan_forbidden_keys() { local doc="$1" local found found=$(jq -r 'paths | .[-1] | tostring' 2>/dev/null <<<"$doc" \ | grep -E "$FORBIDDEN_KEY_REGEX" | head -1 || true) if [ -n "$found" ]; then printf 'forbidden-key:%s' "$found" return 0 fi return 1 } redaction_match_count() { local doc="$1" local total=0 n for pat in "${REDACTION_PATTERNS[@]}"; do n=$(printf '%s' "$doc" | grep -E -c -- "$pat" 2>/dev/null || true) total=$(( total + n )) done printf '%s' "$total" } persisted_artifacts_are_clean() { local path pat for path in "$@"; do [ -e "$path" ] || continue for pat in "${REDACTION_PATTERNS[@]}"; do if [ -d "$path" ]; then grep -R -I -E -q -- "$pat" "$path" 2>/dev/null && return 1 elif grep -I -E -q -- "$pat" "$path" 2>/dev/null; then return 1 fi done done return 0 } validate_schema_fixture() { local schema="$1" jq -e ' ."$schema" == "https://json-schema.org/draft/2020-12/schema" and .type == "object" and .additionalProperties == false and .properties.cases.type == "array" and .properties.cases.items == false and (.properties.cases.prefixItems | length) == 10 and ([.properties.cases.prefixItems[].properties.id.const] | length == 10) and ([.properties.cases.prefixItems[].properties.id.const] | unique | length == 10) and all(.properties.cases.prefixItems[]; ."$ref" == "#/$defs/case" and (.properties.id.const | test("^(claude|pi):(direct|light-pass|repair|write-unavailable|timeout-cancel)$")) and .properties.id.const == (.properties.agent.const + ":" + .properties.scenario.const) and (.properties.outcome.const | IN("completed","error","cancelled")) and (.properties.terminal.const | IN("success","provider_error","cancelled")) and (.properties.cleanup.const | IN("removed","orphan","none")) and .properties.observation.type == "array" and .properties.observation.items == false and (.properties.observation.prefixItems | length > 0) and all(.properties.observation.prefixItems[]; (.properties.stage.const | IN("selector","local","review","cleanup")) and (.properties.outcome.const | IN("observed","failed")) ) ) and (."$defs".case.required | index("process_exit") != null) and ."$defs".case.additionalProperties == false ' "$schema" >/dev/null 2>&1 } validate_manifest() { local schema="$1" doc="$2" validate_schema_fixture "$schema" || return 1 jq -e --slurpfile schema "$schema" ' def digest: type == "string" and test("^sha256:[0-9a-f]{64}$"); def exact_keys($v): (keys | sort) == ($v | sort); ($schema[0].properties.cases.prefixItems | length) as $case_count | exact_keys(["cases","redaction","run_id","runner","runtime","schema_version","source"]) and .schema_version == "1" and (.run_id | digest) and (.source | exact_keys(["head","schema_sha256","script_sha256","source_tree"])) and (.source.head | test("^[0-9a-f]{7,64}$")) and (.source.source_tree | test("^[0-9a-f]{40,64}$")) and (.source.script_sha256 | digest) and (.source.schema_sha256 | digest) and (.runtime | exact_keys(["fixture_sha256","observation_sha256","runtime_sha256","workspace_root_hash"])) and (.runtime.runtime_sha256 | digest) and (.runtime.fixture_sha256 | digest) and (.runtime.observation_sha256 | digest) and (.runtime.workspace_root_hash | digest) and (.runner | exact_keys(["claude_binary_sha256","claude_flags","claude_secret_present","pi_binary_sha256","pi_flags","pi_secret_present"])) and (.runner.claude_binary_sha256 | digest) and (.runner.pi_binary_sha256 | digest) and (.runner.claude_secret_present | type == "boolean") and (.runner.pi_secret_present | type == "boolean") and (.runner.claude_flags == ["--print","--output-format","stream-json","--include-partial-messages","--no-session-persistence","--bare"]) and (.runner.pi_flags == ["--provider","--model","--mode","json","--print","--no-session"]) and (.cases | type == "array" and length == $case_count) and (.redaction | exact_keys(["matches","patterns","sentinels_seeded"])) and .redaction.patterns == ["anthropic_key","pi_key","bearer_value","raw_stdout","raw_prompt"] and (.redaction.sentinels_seeded | type == "number") and .redaction.sentinels_seeded >= 0 and .redaction.matches == 0 ' >/dev/null 2>&1 <<<"$doc" || return 1 local i case_json schema_row derived recorded expected_observation for i in $(seq 0 9); do case_json=$(jq -c --argjson i "$i" '.cases[$i]' <<<"$doc") || return 1 schema_row=$(jq -c --argjson i "$i" '.properties.cases.prefixItems[$i]' "$schema") || return 1 jq -e --argjson row "$schema_row" ' ((keys | sort) == ["agent","argv_hash","cancellation","cleanup","duration_ms","id","observation","outcome","process_exit","scenario","terminal","visible_events","workspace_after","workspace_before"]) and .id == $row.properties.id.const and .agent == $row.properties.agent.const and .scenario == $row.properties.scenario.const and .outcome == $row.properties.outcome.const and .terminal == $row.properties.terminal.const and .cleanup == $row.properties.cleanup.const and (.argv_hash | test("^sha256:[0-9a-f]{64}$")) and (.process_exit | type == "number") and (.process_exit | floor) == .process_exit and .process_exit >= 0 and .process_exit <= 255 and (.duration_ms | type == "number") and (.duration_ms | floor) == .duration_ms and .duration_ms >= 0 and (.visible_events | type == "array" and length > 0) and ([range(0; .visible_events | length)] == [.visible_events[].index]) and all(.visible_events[]; ((keys | sort) == ["detail","index","kind"]) and (.kind | IN("system_init","assistant_text","tool_use","tool_result","partial","terminal_success","terminal_error","terminal_cancelled")) and (.detail | IN("init","text","workspace_write","review_write","repair_write","workspace_cleanup","tool_call","ok","error","event","delta","success","provider_error","cancelled")) ) and all(.observation[]; ((keys | sort) == ["outcome","request_id","stage"]) and (.request_id | test("^rid-[0-9a-f]{8,32}$")) ) and ((.workspace_before | keys | sort) == ["artifacts_present","tree_sha256","writable"]) and ((.workspace_after | keys | sort) == ["artifacts_present","tree_sha256","writable"]) and (.workspace_before.tree_sha256 | test("^sha256:[0-9a-f]{64}$")) and (.workspace_after.tree_sha256 | test("^sha256:[0-9a-f]{64}$")) and (.workspace_before.artifacts_present | type == "boolean") and (.workspace_after.artifacts_present | type == "boolean") and (.workspace_before.writable | type == "boolean") and (.workspace_after.writable | type == "boolean") and ((.cancellation | keys | sort) == ["sentinel_survived","target","triggered"]) and .cancellation.triggered == $row.properties.cancellation.properties.triggered.const and .cancellation.target == $row.properties.cancellation.properties.target.const and (.cancellation.sentinel_survived | type == "boolean") ' >/dev/null 2>&1 <<<"$case_json" || return 1 expected_observation=$(jq -c '[.properties.observation.prefixItems[] | { stage:.properties.stage.const, outcome:.properties.outcome.const }]' <<<"$schema_row") || return 1 jq -e --argjson expected "$expected_observation" ' ([.observation[] | {stage,outcome}] == $expected) ' >/dev/null <<<"$case_json" || return 1 # Each case's observation records must share exactly one runtime request # lifecycle in the closed rid- form (correlation is derived from the log, # not from a predetermined per-case hash). jq -e ' ([.observation[].request_id] | unique | length) == 1 and all(.observation[]; .request_id | test("^rid-[0-9a-f]{8,32}$")) ' >/dev/null <<<"$case_json" || return 1 derived=$(derive_case_result \ "$(jq -r '.agent' <<<"$case_json")" \ "$(jq -r '.scenario' <<<"$case_json")" \ "$(jq -r '.process_exit' <<<"$case_json")" \ "$(jq -r '.cancellation.triggered' <<<"$case_json")" \ "$(jq -r '.cancellation.target' <<<"$case_json")" \ "$(jq -r '.cancellation.sentinel_survived' <<<"$case_json")" \ "$(jq -c '.visible_events' <<<"$case_json")" \ "$(jq -c '.observation' <<<"$case_json")" \ "$(jq -c '.workspace_before' <<<"$case_json")" \ "$(jq -c '.workspace_after' <<<"$case_json")") || return 1 recorded=$(jq -r '[.outcome,.terminal,.cleanup] | join(":")' <<<"$case_json") [ "$derived" = "$recorded" ] || return 1 done scan_forbidden_keys "$doc" >/dev/null 2>&1 && return 1 [ "$(redaction_match_count "$doc")" -eq 0 ] || return 1 return 0 } atomic_write() { local dest="$1" content="$2" local dir dir=$(dirname "$dest") [ -d "$dir" ] || die_validation "output directory absent: $dir" local tmp="$dest.tmp.$$" printf '%s\n' "$content" > "$tmp" mv -f "$tmp" "$dest" } # --------------------------------------------------------------------------- # Top-level modes # --------------------------------------------------------------------------- do_run() { validate_inputs_presence validate_worktree_fingerprint validate_edge_binary_config_fixture_identity validate_runner_and_profile_identity validate_schema_fixture "$FIXTURE_PATH" \ || die_validation "fixture does not implement the closed fixed-matrix schema subset" validate_observation_log_preflight : > "$INVOCATION_MARKER" 2>/dev/null || true RAW_CAPTURE_DIR=$(mktemp -d "$WORKSPACE_ROOT/.e2e-hot-path-capture.XXXXXX") \ || die_validation "cannot create disposable raw capture" if ! run_matrix; then rm -rf "$RAW_CAPTURE_DIR" RAW_CAPTURE_DIR="" die_validation "execution, terminal, cancellation, observation, or workspace evidence contradicted the fixed scenario" fi rm -rf "$RAW_CAPTURE_DIR" RAW_CAPTURE_DIR="" local manifest manifest=$(build_manifest) validate_manifest "$FIXTURE_PATH" "$manifest" \ || die_validation "produced manifest failed supplied schema or runtime correlation validation" persisted_artifacts_are_clean "$WORKSPACE_ROOT" \ || die_validation "surviving workspace artifact contains raw prompt, output, or credential material" atomic_write "$OUTPUT_PATH" "$manifest" log "wrote redacted manifest: $OUTPUT_PATH" } do_preflight() { validate_inputs_presence validate_worktree_fingerprint validate_edge_binary_config_fixture_identity validate_runner_and_profile_identity validate_schema_fixture "$FIXTURE_PATH" \ || die_validation "fixture does not implement the closed fixed-matrix schema subset" validate_observation_log_preflight log "preflight ok" } # --------------------------------------------------------------------------- # Self-test: credential-free behavioral oracle # --------------------------------------------------------------------------- # Pick a writable parent directory whose filesystem permits execve (the default # /tmp is noexec on some sandbox hosts, which would make the fake agent binaries # unrunnable). Respects a caller-supplied TMPDIR first, then falls back to the # repo parent, repo root, HOME, and /var/tmp, probing each with a tiny script. exec_tmp_parent() { local candidate probe for candidate in "${TMPDIR:-/tmp}" "$(dirname "$REPO_ROOT")" "$REPO_ROOT" "${HOME:-}" "/var/tmp"; do [ -n "$candidate" ] || continue [ -d "$candidate" ] || continue [ -w "$candidate" ] || continue probe=$(mktemp -d "$candidate/.e2e-hot-path-probe.XXXXXX" 2>/dev/null) || continue printf '#!/usr/bin/env bash\nexit 0\n' > "$probe/probe" chmod 700 "$probe/probe" if "$probe/probe" >/dev/null 2>&1; then rm -rf "$probe" printf '%s' "$candidate" return 0 fi rm -rf "$probe" done return 1 } write_fake_binary() { local path="$1" agent="$2" cat > "$path" <> "\$marker" 2>/dev/null || true fi record="\${IOP_HOT_PATH_FAKE_RECORD:-}" if [ -n "\$record" ]; then printf '%s\0' "\$@" >> "\$record" 2>/dev/null || true fi agent="\${IOP_HOT_PATH_FAKE_AGENT:-${agent}}" scenario="\${IOP_HOT_PATH_FAKE_SCENARIO:-direct}" rid="\${IOP_HOT_PATH_FAKE_REQUEST_ID:-rid-00000000}" ws="\${IOP_HOT_PATH_FAKE_WORKSPACE:-\$PWD}" contradiction="\${IOP_HOT_PATH_FAKE_CONTRADICTION:-none}" obs_file="\${IOP_HOT_PATH_FAKE_OBSERVATION_FILE:-}" obs_mode="\${IOP_HOT_PATH_FAKE_OBS_MODE:-normal}" obs_write() { # event_class stage attempt disposition reason cleanup orphan request_id [msg] [ -n "\$obs_file" ] || return 0 printf '{"msg":"%s","hot_path_event_class":"%s","hot_path_stage_kind":"%s","hot_path_attempt_bucket":"%s","hot_path_disposition":"%s","hot_path_reason":"%s","hot_path_cleanup_outcome":"%s","hot_path_orphan_outcome":"%s","hot_path_request_id":"%s"}\n' \ "\${9:-hot_path_observation}" "\$1" "\$2" "\$3" "\$4" "\$5" "\$6" "\$7" "\$8" >> "\$obs_file" 2>/dev/null || true } obs_lifecycle() { # \$1=request_id local r="\$1" case "\$scenario" in direct) obs_write dispatch "" "" "" "" "" "" "\$r" obs_write terminal "" "" success "" "" "" "\$r" ;; light-pass) obs_write dispatch "" "" "" "" "" "" "\$r" obs_write stage local first tool_turn "" "" "" "\$r" obs_write stage local retry success "" "" "" "\$r" obs_write light review first "" "" "" "" "\$r" obs_write stage review first tool_turn "" "" "" "\$r" obs_write stage review retry tool_turn "" "" "" "\$r" obs_write stage review retry success "" "" "" "\$r" obs_write light cleanup first "" "" "" "" "\$r" obs_write cleanup "" "" "" "" success "" "\$r" obs_write terminal "" "" success "" "" "" "\$r" ;; repair) obs_write dispatch "" "" "" "" "" "" "\$r" obs_write stage local first tool_turn "" "" "" "\$r" obs_write stage local retry success "" "" "" "\$r" obs_write light review first "" "" "" "" "\$r" obs_write stage review first tool_turn "" "" "" "\$r" obs_write stage review retry tool_turn "" "" "" "\$r" obs_write stage review retry tool_turn "" "" "" "\$r" obs_write light review retry "" "" "" "" "\$r" obs_write stage review retry success "" "" "" "\$r" obs_write light cleanup first "" "" "" "" "\$r" obs_write cleanup "" "" "" "" success "" "\$r" obs_write terminal "" "" success "" "" "" "\$r" ;; write-unavailable) obs_write dispatch "" "" "" provider_error "" "" "\$r" ;; timeout-cancel) obs_write dispatch "" "" "" "" "" "" "\$r" ;; esac } obs_cancel_lifecycle() { [ "\$scenario" = timeout-cancel ] || return 0 obs_write stage local first caller_cancel "" "" "" "\$rid" } # Emit the observation lifecycle BEFORE the stdout events so the timeout-cancel # scenario has already appended its records before it blocks and is signalled. case "\$obs_mode" in none) : ;; rotate) if [ -n "\$obs_file" ]; then mv "\$obs_file" "\$obs_file.rot" 2>/dev/null || true : > "\$obs_file" 2>/dev/null || true fi obs_lifecycle "\$rid" ;; extra-request) obs_lifecycle "\$rid"; obs_write dispatch "" "" "" "" "" "" "rid-otherlifecycle" ;; wrong-stage) obs_write dispatch "" "" "" "" "" "" "\$rid" obs_write stage local first success "" "" "" "\$rid" obs_write terminal "" "" success "" "" "" "\$rid" ;; foreign-message) obs_write dispatch "" "" "" "" "" "" "\$rid" "not_hot_path_observation" ;; unknown-event) obs_write unknown "" "" "" "" "" "" "\$rid" ;; missing-terminal) obs_write dispatch "" "" "" "" "" "" "\$rid" ;; duplicate-terminal) obs_write dispatch "" "" "" "" "" "" "\$rid" obs_write terminal "" "" success "" "" "" "\$rid" obs_write terminal "" "" provider_error "" "" "" "\$rid" ;; late-terminal) obs_lifecycle "\$rid" ( sleep 0.05; obs_write terminal "" "" provider_error "" "" "" "\$rid" ) >/dev/null 2>&1 & ;; cleanup-without-success) obs_write dispatch "" "" "" "" "" "" "\$rid" obs_write cleanup "" "" "" "" primary_error "" "\$rid" obs_write terminal "" "" success "" "" "" "\$rid" ;; unexpected-orphan) obs_write dispatch "" "" "" "" "" "" "\$rid" obs_write orphan "" "" "" "" "" ttl_expired "\$rid" obs_write terminal "" "" success "" "" "" "\$rid" ;; immediate-timeout-orphan) obs_lifecycle "\$rid" if [ "\$scenario" = timeout-cancel ]; then obs_write orphan "" "" "" "" "" ttl_expired "\$rid" fi ;; normal|*) obs_lifecycle "\$rid" ;; esac emit() { printf '%s\n' "\$1"; } emit_artifact() { mkdir -p "\$ws/.iop/job/\$rid" 2>/dev/null || true printf 'plan\n' > "\$ws/.iop/job/\$rid/plan.md" 2>/dev/null || true printf 'review\n' > "\$ws/.iop/job/\$rid/review.md" 2>/dev/null || true } remove_artifact() { rm -rf "\$ws/.iop/job" 2>/dev/null || true } emit_cancelled() { if [ "\$agent" = "claude" ]; then emit '{"type":"result","subtype":"cancelled"}' fi } trap 'obs_cancel_lifecycle; emit_cancelled; exit 143' TERM if [ "\$agent" = "claude" ]; then case "\$scenario" in direct) emit '{"type":"system","subtype":"init"}' if [ "\$contradiction" = "no-terminal" ]; then exit 0; fi emit '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"RAW-OUTPUT-SENTINEL-claude"}]}}' if [ "\$contradiction" = "terminal" ]; then emit '{"type":"result","subtype":"error"}' exit 1 fi emit '{"type":"result","subtype":"success","result":"RAW-OUTPUT-SENTINEL-claude"}' if [ "\$contradiction" = "success-exit" ]; then exit 1; fi ;; light-pass) emit '{"type":"system","subtype":"init"}' emit '{"type":"assistant","message":{"content":[{"type":"text","text":"plan"}]}}' emit_artifact emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"assistant","message":{"content":[{"type":"text","text":"review"}]}}' emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"review_write","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' remove_artifact if [ "\$contradiction" = "empty-reservation" ]; then mkdir -p "\$ws/.iop/job/\$rid"; fi if [ "\$contradiction" != "workspace" ]; then printf 'completed\n' > "\$ws/completed.txt"; fi emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"cleanup_delete","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"result","subtype":"success","result":"done"}' ;; repair) emit '{"type":"system","subtype":"init"}' emit '{"type":"assistant","message":{"content":[{"type":"text","text":"plan"}]}}' emit_artifact emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"assistant","message":{"content":[{"type":"text","text":"defect"}]}}' emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"review_write","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"repair_write","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' printf 'repaired\n' > "\$ws/seeded.txt" remove_artifact if [ "\$contradiction" = "empty-reservation" ]; then mkdir -p "\$ws/.iop/job/\$rid"; fi emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"cleanup_delete","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"result","subtype":"success","result":"repaired"}' ;; write-unavailable) emit '{"type":"system","subtype":"init"}' emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":true}]}}' emit '{"type":"result","subtype":"error","error":"write_unavailable"}' exit 1 ;; timeout-cancel) emit '{"type":"system","subtype":"init"}' emit_artifact emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}' emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}' emit '{"type":"assistant","message":{"content":[{"type":"text","text":"partial"}]}}' if [ "\$contradiction" = "cancel" ]; then emit '{"type":"result","subtype":"success"}' exit 0 fi while :; do sleep 0.1; done ;; esac else case "\$scenario" in direct) emit '{"type":"agent_start"}' emit '{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"RAW-OUTPUT-SENTINEL-pi"}],"stopReason":"stop"},"assistantMessageEvent":{"type":"text_delta"}}' if [ "\$contradiction" = "no-terminal" ]; then exit 0; fi if [ "\$contradiction" = "terminal" ]; then emit '{"type":"message_end","message":{"role":"assistant","content":[],"stopReason":"error"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[],"stopReason":"error"}]}' exit 1 fi emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}' if [ "\$contradiction" = "success-exit" ]; then exit 1; fi ;; light-pass) emit '{"type":"agent_start"}' emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}' emit_artifact emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}' emit '{"type":"tool_execution_start","toolCallId":"tool-review","toolName":"review_write","args":{"path":".iop/job/review.md"}}' emit '{"type":"tool_execution_end","toolCallId":"tool-review","toolName":"review_write","result":{},"isError":false}' remove_artifact if [ "\$contradiction" != "workspace" ]; then printf 'completed\n' > "\$ws/completed.txt"; fi emit '{"type":"tool_execution_start","toolCallId":"tool-cleanup","toolName":"cleanup_delete","args":{"path":".iop/job"}}' emit '{"type":"tool_execution_end","toolCallId":"tool-cleanup","toolName":"cleanup_delete","result":{},"isError":false}' emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}' ;; repair) emit '{"type":"agent_start"}' emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}' emit_artifact emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}' emit '{"type":"tool_execution_start","toolCallId":"tool-review","toolName":"review_write","args":{"path":".iop/job/review.md"}}' emit '{"type":"tool_execution_end","toolCallId":"tool-review","toolName":"review_write","result":{},"isError":false}' emit '{"type":"tool_execution_start","toolCallId":"tool-repair","toolName":"repair_write","args":{"path":"seeded.txt"}}' printf 'repaired\n' > "\$ws/seeded.txt" emit '{"type":"tool_execution_end","toolCallId":"tool-repair","toolName":"repair_write","result":{},"isError":false}' remove_artifact emit '{"type":"tool_execution_start","toolCallId":"tool-cleanup","toolName":"cleanup_delete","args":{"path":".iop/job"}}' emit '{"type":"tool_execution_end","toolCallId":"tool-cleanup","toolName":"cleanup_delete","result":{},"isError":false}' emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"repaired"}],"stopReason":"stop"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"repaired"}],"stopReason":"stop"}]}' ;; write-unavailable) emit '{"type":"agent_start"}' emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}' emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":true}' emit '{"type":"message_end","message":{"role":"assistant","content":[],"stopReason":"error"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[],"stopReason":"error"}]}' exit 0 ;; timeout-cancel) emit '{"type":"agent_start"}' emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}' emit_artifact emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}' emit '{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"partial"}],"stopReason":"stop"},"assistantMessageEvent":{"type":"text_delta"}}' if [ "\$contradiction" = "cancel" ]; then emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}' emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}' exit 0 fi while :; do sleep 0.1; done ;; esac fi exit 0 FAKE_EOF chmod +x "$path" } self_test_assert() { # $1 = label, rest = command; fails the self-test if command exits non-zero. # The command runs in a subshell so that an explicit `exit` (e.g. the exit 69 # from die_validation) terminates only the subshell and can be observed here. local label="$1"; shift if ! ( "$@" ) >/tmp/e2e-hot-path-selftest-out.$$ 2>&1; then cat /tmp/e2e-hot-path-selftest-out.$$ >&2 || true rm -f /tmp/e2e-hot-path-selftest-out.$$ die "self-test assertion failed: $label" fi rm -f /tmp/e2e-hot-path-selftest-out.$$ log "assertion PASS: $label" } self_test_expect_manifest_rejected() { local label="$1" schema="$2" doc="$3" if validate_manifest "$schema" "$doc" >/dev/null 2>&1; then die "self-test assertion failed: $label was accepted" fi log "assertion PASS: $label rejected" } self_test_expect_derive_rejected() { local label="$1"; shift if derive_case_result "$@" >/dev/null 2>&1; then die "self-test assertion failed: $label was accepted" fi log "assertion PASS: $label rejected" } self_test_expect_run_rejected() { local label="$1" rc=0 rm -f "$OUTPUT_PATH" ( do_run ) >/tmp/e2e-hot-path-negative.$$ 2>&1 || rc=$? rm -f /tmp/e2e-hot-path-negative.$$ [ "$rc" -eq "$EXIT_VALIDATION" ] \ || die "self-test assertion failed: $label should exit 69 (got $rc)" [ ! -e "$OUTPUT_PATH" ] \ || die "self-test assertion failed: $label wrote a manifest" log "assertion PASS: $label rejected before manifest output" } self_test_expect_preinvocation_reject() { local label="$1" rc=0 : > "$SELF_TEST_MARKER" ( do_run ) >/tmp/e2e-hot-path-preinv.$$ 2>&1 || rc=$? rm -f /tmp/e2e-hot-path-preinv.$$ [ "$rc" -eq "$EXIT_VALIDATION" ] \ || die "self-test assertion failed: $label should exit 69 (got $rc)" [ ! -s "$SELF_TEST_MARKER" ] \ || die "self-test assertion failed: $label invoked an agent before identity validation" log "assertion PASS: $label rejected before invocation" } self_test() { require_cmd jq require_cmd sha256sum require_cmd grep require_cmd timeout local tmp_parent root tmp_parent=$(exec_tmp_parent) \ || die "no writable+executable temp parent found; set TMPDIR to an executable dir" root=$(mktemp -d "$tmp_parent/e2e-hot-path-self-test.XXXXXX") # Ensure all temporary state is removed on any exit (success or failure). SELF_TEST_ROOT="$root" trap 'rm -rf "$SELF_TEST_ROOT"' EXIT local bin_dir="$root/bin" ws_root="$root/ws" local claude_bin="$bin_dir/fake-claude" pi_bin="$bin_dir/fake-pi" local edge_bin="$bin_dir/fake-edge" edge_config="$root/edge.yaml" local pi_config_dir="$root/pi-config" local obs_file="$root/hot-path-observation.log" local runtime_ev="$root/runtime-evidence.json" local out="$root/manifest.json" local marker="$root/invocation.marker" mkdir -p "$bin_dir" "$ws_root" "$pi_config_dir" SELF_TEST_MARKER="$marker" INVOCATION_MARKER="$marker" SENTINELS_SEEDED=4 REQUIRE_RECORDED_ARGV=true write_fake_binary "$claude_bin" claude write_fake_binary "$pi_bin" pi # A fake Edge binary/config and Pi config dir stand in for the real runtime # identity inputs. They are never executed by the self-test. printf '#!/usr/bin/env bash\nexit 0\n' > "$edge_bin"; chmod +x "$edge_bin" printf 'edge:\n hot_path:\n enabled: true\n' > "$edge_config" printf 'provider: iop-pi-smoke\nbase_url: fake\n' > "$pi_config_dir/config.yaml" : > "$obs_file" # Sentinel secret env values (presence-only; never serialized). export IOP_FAKE_CLAUDE_KEY='sk-ant-fake-CLAUDE-SENTINEL-0' export IOP_FAKE_PI_KEY='pi-fake-PI-SENTINEL-0' # Non-secret base/profile/alias identity inputs (fake; never contacted). local base_url="https://iop-hot-smoke.invalid/v1" local provider="iop-pi-smoke" local direct_model="iop-preset-direct" local pass_model="iop-preset-pass" local repair_model="iop-preset-repair" local slow_model="iop-preset-slow" # Actual identity digests the harness will recompute and compare. local script_sha schema_sha head tree fp script_sha=$(sha256_file "$SELF_PATH") schema_sha=$(sha256_file "$SCHEMA_PATH") head=$(git_head) tree=$(git_tree) # Compute the worktree fingerprint once at top level and export it so every # `( do_run )` / `( do_preflight )` subshell inherits the cache instead of # re-traversing the tree. WORKTREE_FINGERPRINT_CACHE=$(compute_worktree_fingerprint) export WORKTREE_FINGERPRINT_CACHE fp="$WORKTREE_FINGERPRINT_CACHE" local claude_sha pi_sha edge_sha edge_cfg_sha pi_cfg_sha fixture_sha claude_sha=$(sha256_file "$claude_bin") pi_sha=$(sha256_file "$pi_bin") edge_sha=$(sha256_file "$edge_bin") edge_cfg_sha=$(sha256_file "$edge_config") pi_cfg_sha=$(tree_sha256 "$pi_config_dir") fixture_sha="$schema_sha" local base_sha provider_sha direct_sha pass_sha repair_sha slow_sha base_sha=$(sha256_str "$base_url") provider_sha=$(sha256_str "$provider") direct_sha=$(sha256_str "$direct_model") pass_sha=$(sha256_str "$pass_model") repair_sha=$(sha256_str "$repair_model") slow_sha=$(sha256_str "$slow_model") jq -n \ --arg script_sha256 "$script_sha" --arg schema_sha256 "$schema_sha" \ --arg head "$head" --arg source_tree "$tree" --arg worktree_fingerprint "$fp" \ --arg claude_binary_sha256 "$claude_sha" --arg pi_binary_sha256 "$pi_sha" \ --arg edge_binary_sha256 "$edge_sha" --arg edge_config_sha256 "$edge_cfg_sha" \ --arg pi_config_sha256 "$pi_cfg_sha" --arg fixture_sha256 "$fixture_sha" \ --arg base_url_sha256 "$base_sha" --arg pi_provider_sha256 "$provider_sha" \ --arg direct_model_sha256 "$direct_sha" --arg pass_model_sha256 "$pass_sha" \ --arg repair_model_sha256 "$repair_sha" --arg slow_model_sha256 "$slow_sha" \ '{ script_sha256:$script_sha256, schema_sha256:$schema_sha256, head:$head, source_tree:$source_tree, worktree_fingerprint:$worktree_fingerprint, claude_binary_sha256:$claude_binary_sha256, pi_binary_sha256:$pi_binary_sha256, edge_binary_sha256:$edge_binary_sha256, edge_config_sha256:$edge_config_sha256, pi_config_sha256:$pi_config_sha256, fixture_sha256:$fixture_sha256, base_url_sha256:$base_url_sha256, pi_provider_sha256:$pi_provider_sha256, direct_model_sha256:$direct_model_sha256, pass_model_sha256:$pass_model_sha256, repair_model_sha256:$repair_model_sha256, slow_model_sha256:$slow_model_sha256 }' > "$runtime_ev" local bad_digest="sha256:0000000000000000000000000000000000000000000000000000000000000000" local ev_fp_bad="$root/ev-fp-bad.json" ev_claude_bad="$root/ev-claude-bad.json" local ev_edge_bad="$root/ev-edge-bad.json" ev_edge_cfg_bad="$root/ev-edge-cfg-bad.json" local ev_pi_cfg_bad="$root/ev-pi-cfg-bad.json" ev_base_bad="$root/ev-base-bad.json" local ev_model_bad="$root/ev-model-bad.json" ev_fixture_bad="$root/ev-fixture-bad.json" jq --arg b "$bad_digest" '.worktree_fingerprint=$b' "$runtime_ev" > "$ev_fp_bad" jq --arg b "$bad_digest" '.claude_binary_sha256=$b' "$runtime_ev" > "$ev_claude_bad" jq --arg b "$bad_digest" '.edge_binary_sha256=$b' "$runtime_ev" > "$ev_edge_bad" jq --arg b "$bad_digest" '.edge_config_sha256=$b' "$runtime_ev" > "$ev_edge_cfg_bad" jq --arg b "$bad_digest" '.pi_config_sha256=$b' "$runtime_ev" > "$ev_pi_cfg_bad" jq --arg b "$bad_digest" '.base_url_sha256=$b' "$runtime_ev" > "$ev_base_bad" jq --arg b "$bad_digest" '.slow_model_sha256=$b' "$runtime_ev" > "$ev_model_bad" jq --arg b "$bad_digest" '.fixture_sha256=$b' "$runtime_ev" > "$ev_fixture_bad" local -a good_inputs=( --claude "$claude_bin" --pi "$pi_bin" --runtime-evidence "$runtime_ev" --fixture "$SCHEMA_PATH" --base-url "$base_url" --direct-model "$direct_model" --pass-model "$pass_model" --repair-model "$repair_model" --slow-model "$slow_model" --edge-bin "$edge_bin" --edge-config "$edge_config" --pi-config-dir "$pi_config_dir" --pi-provider "$provider" --observation-file "$obs_file" --workspace-root "$ws_root" --output "$out" --claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY ) # --- Positive run through the shared --run path with fake binaries. --- : > "$obs_file" parse_run_inputs "${good_inputs[@]}" self_test_assert "positive do_run exits 0" do_run local manifest manifest=$(cat "$out") # --- Manifest validation (shared validator used by --run). --- self_test_assert "produced manifest validates against supplied fixture" \ validate_manifest "$SCHEMA_PATH" "$manifest" self_test_assert "production retry observation traces accepted and reduced" \ bash -c "jq -e 'all(.cases[] | select(.scenario==\"light-pass\" or .scenario==\"repair\"); [.observation[].stage] == [\"selector\",\"local\",\"review\",\"cleanup\"])' <<<\"\$1\" >/dev/null" _ "$manifest" self_test_assert "native Pi success and error terminals parsed" \ bash -c "jq -e '(.cases[] | select(.id==\"pi:direct\") | .terminal==\"success\") and (.cases[] | select(.id==\"pi:write-unavailable\") | .terminal==\"provider_error\")' <<<\"\$1\" >/dev/null" _ "$manifest" self_test_assert "native Pi JSON error with exit 0 accepted" \ bash -c "jq -e '.cases[] | select(.id==\"pi:write-unavailable\") | .process_exit==0 and .terminal==\"provider_error\" and .outcome==\"error\"' <<<\"\$1\" >/dev/null" _ "$manifest" self_test_assert "Pi terminal error with exit 0 derivation accepted" \ derive_case_result pi write-unavailable 0 false none true \ '[{"index":0,"kind":"tool_use","detail":"workspace_write"},{"index":1,"kind":"tool_result","detail":"error"},{"index":2,"kind":"terminal_error","detail":"provider_error"}]' \ '[{"request_id":"rid-deadbeef","stage":"selector","outcome":"failed"}]' \ '{"artifacts_present":false,"writable":false,"tree_sha256":"sha256:before"}' \ '{"artifacts_present":false,"writable":false,"tree_sha256":"sha256:before"}' self_test_expect_derive_rejected "Pi success terminal with nonzero exit" \ pi direct 1 false none true \ '[{"index":0,"kind":"assistant_text","detail":"text"},{"index":1,"kind":"terminal_success","detail":"success"}]' \ '[{"request_id":"rid-deadbeef","stage":"selector","outcome":"observed"}]' \ '{"artifacts_present":false,"writable":true,"tree_sha256":"sha256:before"}' \ '{"artifacts_present":false,"writable":true,"tree_sha256":"sha256:before"}' self_test_assert "native Pi signal exit 143 reconciled as cancellation" \ bash -c "jq -e '.cases[] | select(.id==\"pi:timeout-cancel\") | .process_exit==143 and .terminal==\"cancelled\" and .cancellation.target==\"child_only\"' <<<\"\$1\" >/dev/null" _ "$manifest" self_test_assert "native Pi scenario tool order is visible" \ bash -c "jq -e '.cases[] | select(.id==\"pi:repair\") | [.visible_events[] | select(.kind==\"tool_use\") | .detail] == [\"workspace_write\",\"review_write\",\"repair_write\",\"workspace_cleanup\"]' <<<\"\$1\" >/dev/null" _ "$manifest" # --- Exactly the ten expected case ids in matrix order. --- local ids expected_ids ids=$(jq -r '.cases[].id' <<<"$manifest") expected_ids=$(printf '%s\n' "${EXPECTED_CASE_IDS[@]}") self_test_assert "ten unique case ids" \ bash -c '[ "$1" = "$2" ]' _ "$ids" "$expected_ids" # Exact argv comparison occurred inside every case before disposable raw # capture was deleted. No expected/recorded argv or raw observation fragment # may survive. self_test_assert "raw argv/stdout/observation capture deleted" \ bash -c '! find "$1" -name "argv-*" -o -name "out-*.jsonl" -o -name "obs-appended-*" | grep -q .' _ "$root" # --- Observations are projected per case from the appended log region. --- self_test_assert "observation request ids projected and single per case" \ bash -c "jq -e 'all(.cases[]; ([.observation[].request_id]|unique|length)==1 and all(.observation[]; .request_id|test(\"^rid-[0-9a-f]{8,32}\$\")))' <<<\"\$1\" >/dev/null" _ "$manifest" # --- Success and expected-failure terminals. --- self_test_assert "direct cases terminal=success" \ bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"direct\")|.terminal' <<<\"\$1\" | sort -u)\" = \"success\" ]" _ "$manifest" self_test_assert "write-unavailable terminal=provider_error" \ bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"write-unavailable\")|.terminal' <<<\"\$1\" | sort -u)\" = \"provider_error\" ]" _ "$manifest" self_test_assert "timeout-cancel terminal=cancelled" \ bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"timeout-cancel\")|.terminal' <<<\"\$1\" | sort -u)\" = \"cancelled\" ]" _ "$manifest" self_test_assert "process exit status is captured from wait" \ bash -c "jq -e 'all(.cases[]|select(.terminal==\"success\"); .process_exit==0) and all(.cases[]|select(.agent==\"claude\" and .terminal==\"provider_error\"); .process_exit!=0) and all(.cases[]|select(.agent==\"pi\" and .terminal==\"provider_error\"); .process_exit==0) and all(.cases[]|select(.terminal==\"cancelled\"); .process_exit!=0)' <<<\"\$1\" >/dev/null" _ "$manifest" # --- Cleanup/orphan classification. --- self_test_assert "light-pass/repair cleanup=removed" \ bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"light-pass\" or .scenario==\"repair\")|.cleanup' <<<\"\$1\" | sort -u)\" = \"removed\" ]" _ "$manifest" self_test_assert "timeout-cancel cleanup=orphan" \ bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"timeout-cancel\")|.cleanup' <<<\"\$1\" | sort -u)\" = \"orphan\" ]" _ "$manifest" # --- Child-only timeout signaling. --- self_test_assert "timeout-cancel child_only target" \ bash -c "jq -e '.cases[]|select(.scenario==\"timeout-cancel\")|.cancellation.target==\"child_only\" and .cancellation.sentinel_survived==true' <<<\"\$1\" >/dev/null" _ "$manifest" # --- Secret absence / zero-match redaction over the real manifest. --- self_test_assert "redaction matches == 0 on manifest" \ bash -c "[ \"\$(grep -E -c -- 'sk-ant-[A-Za-z0-9_-]+|pi-fake-PI-SENTINEL-[0-9]+|IOP_FAKE_CLAUDE_KEY|IOP_FAKE_PI_KEY|Bearer[ ]?[A-Za-z0-9._-]+' <<<\"\$1\" || true)\" = \"0\" ]" _ "$manifest" # --- Redaction is non-vacuous: a leaked sentinel is detected. --- local leak leak='{"runner":{"note":"sk-ant-fake-CLAUDE-SENTINEL-0 leaked"}}' self_test_assert "redaction detects leaked sentinel" \ bash -c "[ \"\$(grep -E -c -- 'sk-ant-[A-Za-z0-9_-]+|pi-fake-PI-SENTINEL-[0-9]+' <<<\"\$1\" || true)\" != \"0\" ]" _ "$leak" self_test_assert "all surviving harness artifacts are redacted" \ persisted_artifacts_are_clean "$ws_root" "$out" "$obs_file" local content_probe="$root/content-probe" content_before content_after mkdir -p "$content_probe" printf 'before\n' > "$content_probe/same-name.txt" content_before=$(tree_sha256 "$content_probe") printf 'after\n' > "$content_probe/same-name.txt" content_after=$(tree_sha256 "$content_probe") self_test_assert "workspace digest changes on content-only edit" \ bash -c '[ "$1" != "$2" ]' _ "$content_before" "$content_after" # --- Schema rejection: a malformed manifest must fail validation. --- local bad_manifest bad_manifest=$(jq '.cases |= .[0:9]' <<<"$manifest") # only 9 cases self_test_expect_manifest_rejected "9-case manifest" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].prompt = "raw"' <<<"$manifest") # forbidden field self_test_expect_manifest_rejected "forbidden-field manifest" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].outcome = "bogus"' <<<"$manifest") # bad enum self_test_expect_manifest_rejected "bad-enum manifest" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases += [.cases[0]]' <<<"$manifest") # 11 cases / duplicate id self_test_expect_manifest_rejected "11-case duplicate manifest" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[1].id = .cases[0].id' <<<"$manifest") self_test_expect_manifest_rejected "distinct-row duplicate id" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].agent = "pi"' <<<"$manifest") self_test_expect_manifest_rejected "id-agent mismatch" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].scenario = "repair"' <<<"$manifest") self_test_expect_manifest_rejected "id-scenario mismatch" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].terminal = "provider_error"' <<<"$manifest") self_test_expect_manifest_rejected "terminal-event contradiction" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[0].cancellation.triggered = true' <<<"$manifest") self_test_expect_manifest_rejected "cancellation relation mismatch" "$SCHEMA_PATH" "$bad_manifest" bad_manifest=$(jq '.cases[1].observation[0].request_id = "rid-deadbeef00"' <<<"$manifest") self_test_expect_manifest_rejected "multi-request observation in one case" "$SCHEMA_PATH" "$bad_manifest" local alternate_fixture="$root/alternate-schema.json" malformed_fixture="$root/malformed-schema.json" jq '.properties.cases.prefixItems[0].properties.id.const = "pi:direct"' \ "$SCHEMA_PATH" > "$alternate_fixture" self_test_expect_manifest_rejected "alternate fixture changes acceptance" "$alternate_fixture" "$manifest" jq '.properties.cases.prefixItems |= .[0:9]' "$SCHEMA_PATH" > "$malformed_fixture" self_test_expect_manifest_rejected "malformed nine-row fixture" "$malformed_fixture" "$manifest" # --- Identity mismatches exit 69 before any agent invocation (R1). --- local ev for ev in \ "worktree fingerprint mismatch:$ev_fp_bad" \ "claude binary identity mismatch:$ev_claude_bad" \ "edge binary identity mismatch:$ev_edge_bad" \ "edge config identity mismatch:$ev_edge_cfg_bad" \ "pi config identity mismatch:$ev_pi_cfg_bad" \ "base url identity mismatch:$ev_base_bad" \ "scenario alias identity mismatch:$ev_model_bad" \ "fixture identity mismatch:$ev_fixture_bad"; do local label="${ev%%:*}" ev_file="${ev##*:}" : > "$obs_file" parse_run_inputs \ --claude "$claude_bin" --pi "$pi_bin" \ --runtime-evidence "$ev_file" --fixture "$SCHEMA_PATH" \ --base-url "$base_url" \ --direct-model "$direct_model" --pass-model "$pass_model" \ --repair-model "$repair_model" --slow-model "$slow_model" \ --edge-bin "$edge_bin" --edge-config "$edge_config" \ --pi-config-dir "$pi_config_dir" --pi-provider "$provider" \ --observation-file "$obs_file" --workspace-root "$ws_root" \ --output "$out" \ --claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY self_test_expect_preinvocation_reject "$label" done # --- Observation lifecycle and freshness negative controls (R2). --- parse_run_inputs "${good_inputs[@]}" local saved_observation_wait="$OBSERVATION_WAIT_MSEC" OBSERVATION_WAIT_MSEC=250 : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=none self_test_expect_run_rejected "post-bound lifecycle timeout" # Stale-only: valid-looking records exist before the case offset but nothing # is appended for the current case; the run must reject the stale evidence. : > "$obs_file" printf '{"msg":"hot_path_observation","hot_path_event_class":"dispatch","hot_path_stage_kind":"","hot_path_reason":"","hot_path_request_id":"%s"}\n' \ "$(request_id_for claude:direct)" >> "$obs_file" self_test_expect_run_rejected "stale-only observation rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=rotate self_test_expect_run_rejected "rotated/truncated observation rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=extra-request self_test_expect_run_rejected "mixed/duplicate request lifecycle rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=wrong-stage self_test_expect_run_rejected "wrong observation stage lifecycle rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=foreign-message self_test_expect_run_rejected "foreign-message observation lookalike rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=unknown-event self_test_expect_run_rejected "unknown production observation event rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=missing-terminal self_test_expect_run_rejected "missing observation terminal rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=duplicate-terminal self_test_expect_run_rejected "duplicate conflicting observation terminals rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=late-terminal self_test_expect_run_rejected "late contradictory observation terminal rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=cleanup-without-success self_test_expect_run_rejected "cleanup without successful lifecycle rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=unexpected-orphan self_test_expect_run_rejected "unexpected observation orphan rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE : > "$obs_file" export IOP_HOT_PATH_FAKE_OBS_MODE=immediate-timeout-orphan self_test_expect_run_rejected "immediate TTL orphan after caller cancellation rejected" unset IOP_HOT_PATH_FAKE_OBS_MODE OBSERVATION_WAIT_MSEC="$saved_observation_wait" # --- Execution/terminal/workspace contradictions exit 69 (retained). --- : > "$obs_file" local false_runtime_ev="$root/runtime-evidence-false.json" false_sha false_sha=$(sha256_file /bin/false) jq --arg c "$false_sha" --arg p "$false_sha" \ '.claude_binary_sha256=$c | .pi_binary_sha256=$p' "$runtime_ev" > "$false_runtime_ev" parse_run_inputs \ --claude /bin/false --pi /bin/false \ --runtime-evidence "$false_runtime_ev" --fixture "$SCHEMA_PATH" \ --base-url "$base_url" \ --direct-model "$direct_model" --pass-model "$pass_model" \ --repair-model "$repair_model" --slow-model "$slow_model" \ --edge-bin "$edge_bin" --edge-config "$edge_config" \ --pi-config-dir "$pi_config_dir" --pi-provider "$provider" \ --observation-file "$obs_file" --workspace-root "$ws_root" \ --output "$out" \ --claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY self_test_expect_run_rejected "immediate exit with no native output" parse_run_inputs "${good_inputs[@]}" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=no-terminal self_test_expect_run_rejected "missing native terminal" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=terminal self_test_expect_run_rejected "terminal and scenario contradiction" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=success-exit self_test_expect_run_rejected "success terminal with nonzero exit rejected" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=workspace self_test_expect_run_rejected "content-insensitive cleanup contradiction" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=empty-reservation self_test_expect_run_rejected "empty reserved request directory rejected" : > "$obs_file" export IOP_HOT_PATH_FAKE_CONTRADICTION=cancel self_test_expect_run_rejected "timeout without triggered child cancellation" unset IOP_HOT_PATH_FAKE_CONTRADICTION # Native Pi rejects the old OpenAI-choice lookalike and an agent_end that # lacks a terminal-capable assistant message. local pi_choices_probe="$root/pi-choices-lookalike.jsonl" local pi_bad_end_probe="$root/pi-agent-end-without-assistant.jsonl" pi_probe_events printf '%s\n' '{"choices":[{"finish_reason":"stop"}]}' > "$pi_choices_probe" pi_probe_events=$(parse_visible_events pi "$pi_choices_probe" 0 false none) if jq -e 'any(.[]; .kind | startswith("terminal_"))' <<<"$pi_probe_events" >/dev/null; then die "self-test assertion failed: OpenAI choices lookalike produced a Pi terminal" fi log "assertion PASS: OpenAI choices lookalike rejected for Pi" printf '%s\n' '{"type":"agent_start"}' '{"type":"agent_end","messages":[]}' > "$pi_bad_end_probe" if parse_visible_events pi "$pi_bad_end_probe" 0 false none >/dev/null 2>&1; then die "self-test assertion failed: Pi agent_end without assistant was accepted" fi log "assertion PASS: Pi agent_end without terminal-capable assistant rejected" # --- Preflight validates without invoking agents. --- : > "$obs_file" rm -f "$marker" parse_run_inputs "${good_inputs[@]}" self_test_assert "preflight ok" do_preflight if [ -f "$marker" ] && [ -s "$marker" ]; then die "self-test assertion failed: preflight invoked an agent" fi # --- Removal of all temporary state. --- rm -rf "$root" if [ -d "$root" ]; then die "self-test assertion failed: temporary state was not removed" fi log "self-test PASSED: exact argv, fixed 2x5 matrix, schema rejection," log " runtime/profile/alias binding mismatch exit 69 before invocation," log " production retry lifecycle closure and negative observation controls," log " native Pi success/error/cancel plus tool order, empty-reservation" log " rejection, secret absence, child-only cancellation, cleanup/orphan" log " classification, and full cleanup verified with fake agents/runtime only." return 0 } main() { local mode="${1:-}" case "$mode" in --self-test) self_test ;; --preflight-only) shift parse_run_inputs "$@" INVOCATION_MARKER="${IOP_HOT_PATH_INVOCATION_MARKER:-/dev/null}" SENTINELS_SEEDED=0 REQUIRE_RECORDED_ARGV=false do_preflight ;; --run) shift parse_run_inputs "$@" INVOCATION_MARKER="${IOP_HOT_PATH_INVOCATION_MARKER:-/dev/null}" SENTINELS_SEEDED=0 REQUIRE_RECORDED_ARGV=false do_run ;; -h|--help) usage; exit "$EXIT_OK" ;; *) usage; exit "$EXIT_USAGE" ;; esac } main "$@"