2083 lines
102 KiB
Bash
Executable file
2083 lines
102 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# scripts/e2e-hot-path-agents.sh
|
|
#
|
|
# Secret-safe Claude/Pi Hot Path smoke harness.
|
|
#
|
|
# Modes:
|
|
# --self-test Credential-free behavioral oracle. Builds fake Claude/Pi
|
|
# binaries, a fake Edge binary/config, a fake Pi config dir,
|
|
# runtime identity evidence, a live observation log, disposable
|
|
# workspaces and sentinel secrets under one mktemp -d, then
|
|
# runs the fixed 2x5 matrix through the same manifest builder
|
|
# and validator used by --run and asserts every safety proof,
|
|
# including runtime/profile binding and fresh-observation
|
|
# rejection.
|
|
# --preflight-only Validate non-secret inputs, current worktree fingerprint,
|
|
# Edge/Pi/CLI runtime identity, base/profile/alias binding and
|
|
# the observation log without invoking any agent.
|
|
# --run Validate inputs/identity, bind both CLIs to the supplied IOP
|
|
# base/profile and per-scenario preset alias, run the fixed
|
|
# {claude,pi} x {direct,light-pass,repair,write-unavailable,
|
|
# timeout-cancel} matrix in disposable workspaces while
|
|
# capturing only freshly appended observation-log records, and
|
|
# atomically emit a redacted caller-supplied manifest.
|
|
#
|
|
# This harness never prints secret, endpoint, config, or model values. Missing or
|
|
# mismatched source/worktree/runtime/config/binary/fixture/base/profile/alias
|
|
# facts exit 69 before any agent invocation. Each case consumes only observation
|
|
# records appended by the selected runtime after that case started; stale,
|
|
# rotated, truncated, missing, mixed, or wrong-stage evidence is rejected. No
|
|
# Makefile, deployment, shared-process, or tracked smoke output is touched. The
|
|
# self-test path does not contact the network and does not invoke the installed
|
|
# Pi/Claude/Edge or any provider.
|
|
set -euo pipefail
|
|
|
|
readonly EXIT_OK=0
|
|
readonly EXIT_USAGE=64
|
|
readonly EXIT_VALIDATION=69
|
|
readonly EXIT_SOFTWARE=70
|
|
|
|
readonly SCHEMA_VERSION="1"
|
|
readonly EXIT_TIMEOUT=124
|
|
|
|
# Exact pinned adapter argv (the prompt and provider/model identity are appended
|
|
# by the adapter builders; the base/model is bound through the environment and is
|
|
# never serialized).
|
|
readonly CLAUDE_FLAGS=(--print --output-format stream-json --include-partial-messages --no-session-persistence --bare)
|
|
readonly PI_FLAGS=(--provider --model --mode json --print --no-session)
|
|
|
|
readonly AGENTS=(claude pi)
|
|
readonly SCENARIOS=(direct light-pass repair write-unavailable timeout-cancel)
|
|
readonly EXPECTED_CASE_IDS=(
|
|
claude:direct
|
|
claude:light-pass
|
|
claude:repair
|
|
claude:write-unavailable
|
|
claude:timeout-cancel
|
|
pi:direct
|
|
pi:light-pass
|
|
pi:repair
|
|
pi:write-unavailable
|
|
pi:timeout-cancel
|
|
)
|
|
|
|
# Deterministic worktree fingerprint input set (SDD S16 runtime/source identity).
|
|
# A content change to any of these paths changes the fingerprint without exposing
|
|
# any file value.
|
|
readonly WORKTREE_FINGERPRINT_PATHS=(
|
|
apps/edge
|
|
packages/go/streamgate
|
|
packages/go/config
|
|
scripts/e2e-hot-path-agents.sh
|
|
scripts/fixtures/hot-path-agent-smoke-manifest.schema.json
|
|
go.mod
|
|
go.sum
|
|
)
|
|
|
|
# Forbidden manifest field names and redaction patterns. The schema rejects these
|
|
# names via patternProperties->false and closed objects; validate_manifest scans
|
|
# recursively as a defense-in-depth check.
|
|
readonly FORBIDDEN_KEY_REGEX='^(prompt|output|token|key|auth|credential|secret|password|api_key|apikey|endpoint|bearer|cookie|session_token)$'
|
|
readonly REDACTION_PATTERNS=(
|
|
'sk-ant-[A-Za-z0-9_-]+'
|
|
'pi-fake-PI-SENTINEL-[0-9]+'
|
|
'Bearer[ ]?[A-Za-z0-9._-]+'
|
|
'RAW-OUTPUT-SENTINEL-[A-Za-z0-9_-]+'
|
|
'Summarize the workspace README'
|
|
'Author the plan/review pair'
|
|
'The seeded file has a defect'
|
|
'Perform a long running analysis'
|
|
)
|
|
readonly REDACTION_PATTERN_LABELS=(
|
|
anthropic_key
|
|
pi_key
|
|
bearer_value
|
|
raw_stdout
|
|
raw_prompt
|
|
)
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SCHEMA_PATH="$SCRIPT_DIR/fixtures/hot-path-agent-smoke-manifest.schema.json"
|
|
SELF_PATH="$SCRIPT_DIR/e2e-hot-path-agents.sh"
|
|
|
|
# Timeout (seconds) for the timeout-cancel scenario before child-only signaling.
|
|
readonly CANCEL_TIMEOUT_SEC=1
|
|
readonly SHARED_SENTINEL_LIFE_SEC=5
|
|
OBSERVATION_WAIT_MSEC=5000
|
|
OBSERVATION_CANCEL_WAIT_MSEC=10000
|
|
OBSERVATION_QUIET_MSEC=150
|
|
|
|
log() { printf '[e2e-hot-path-agents] %s\n' "$*" >&2; }
|
|
die() { log "error: $*"; exit "${EXIT_SOFTWARE}"; }
|
|
die_usage() { log "usage: $*"; exit "${EXIT_USAGE}"; }
|
|
die_validation() { log "validation failed: $*"; exit "${EXIT_VALIDATION}"; }
|
|
|
|
require_cmd() {
|
|
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
|
}
|
|
|
|
sha256_str() {
|
|
printf '%s' "$1" | sha256sum | awk '{printf "sha256:%s", $1}'
|
|
}
|
|
|
|
sha256_file() {
|
|
local p="$1"
|
|
[ -f "$p" ] || die "sha256_file: missing file: $p"
|
|
sha256sum "$p" | awk '{printf "sha256:%s", $1}'
|
|
}
|
|
|
|
# Hash sorted relative paths, file sizes, and file bytes. A content-only change
|
|
# therefore changes the digest without exposing any workspace value.
|
|
tree_sha256() {
|
|
local dir="$1"
|
|
[ -d "$dir" ] || die "tree_sha256: missing dir: $dir"
|
|
(
|
|
cd "$dir" || exit 1
|
|
while IFS= read -r -d '' path; do
|
|
printf 'path:%s\0size:%s\0' "$path" "$(stat -c '%s' -- "$path")"
|
|
sha256sum -- "$path" | awk '{printf "content:%s\0", $1}'
|
|
done < <(find . -type f -printf '%P\0' 2>/dev/null | LC_ALL=C sort -z)
|
|
) | sha256sum | awk '{printf "sha256:%s", $1}'
|
|
}
|
|
|
|
git_head() {
|
|
git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null \
|
|
|| printf '0000000000000000000000000000000000000000'
|
|
}
|
|
|
|
git_tree() {
|
|
git -C "$REPO_ROOT" rev-parse "HEAD:scripts" 2>/dev/null \
|
|
|| printf '0000000000000000000000000000000000000000'
|
|
}
|
|
|
|
# Compute a deterministic digest over the current worktree inputs (tracked and
|
|
# untracked bytes) that back the Hot Path runtime, so external identity binds to
|
|
# the exact checkout rather than HEAD-only identity. Each per-file `sha256sum`
|
|
# line carries both content and path, so a content or path change flips the
|
|
# digest; hashing is batched through xargs so the traversal stays cheap even for
|
|
# large directories. Never prints file content.
|
|
compute_worktree_fingerprint() {
|
|
(
|
|
cd "$REPO_ROOT" || exit 1
|
|
{
|
|
local p
|
|
for p in "${WORKTREE_FINGERPRINT_PATHS[@]}"; do
|
|
if [ -d "$p" ]; then
|
|
find "$p" -type f -print0 2>/dev/null
|
|
elif [ -f "$p" ]; then
|
|
printf '%s\0' "$p"
|
|
fi
|
|
done
|
|
} | LC_ALL=C sort -z | xargs -0 -r sha256sum
|
|
) | sha256sum | awk '{printf "sha256:%s", $1}'
|
|
}
|
|
|
|
# Memoized worktree fingerprint: the worktree does not change within one run, so
|
|
# the (potentially large) traversal happens at most once.
|
|
worktree_fingerprint() {
|
|
if [ -z "${WORKTREE_FINGERPRINT_CACHE:-}" ]; then
|
|
WORKTREE_FINGERPRINT_CACHE=$(compute_worktree_fingerprint)
|
|
fi
|
|
printf '%s' "$WORKTREE_FINGERPRINT_CACHE"
|
|
}
|
|
|
|
# Build the exact Claude argv tokens (excluding the binary path). The prompt is
|
|
# positional; the workspace is supplied via the process working directory and the
|
|
# base/model are supplied via the environment, so neither becomes an argv token.
|
|
build_claude_argv() {
|
|
local prompt="$1"
|
|
printf '%s\0' "${CLAUDE_FLAGS[@]}" "$prompt"
|
|
}
|
|
|
|
# Build the exact Pi argv tokens (excluding the binary path).
|
|
build_pi_argv() {
|
|
local provider="$1" model="$2" prompt="$3"
|
|
printf '%s\0' \
|
|
"--provider" "$provider" \
|
|
"--model" "$model" \
|
|
"--mode" "json" \
|
|
"--print" \
|
|
"--no-session" \
|
|
"$prompt"
|
|
}
|
|
|
|
request_id_for() {
|
|
local case_id="$1"
|
|
printf 'rid-%s' "$(sha256_str "$case_id" | sed 's/^sha256://' | cut -c1-8)"
|
|
}
|
|
|
|
# Deterministic scenario -> preset alias map. Each of the five scenarios binds to
|
|
# exactly one of the four caller-supplied model aliases so a structurally valid
|
|
# run must reach the intended IOP preset rather than an arbitrary host default.
|
|
scenario_model_alias() {
|
|
case "$1" in
|
|
direct) printf '%s' "$DIRECT_MODEL" ;;
|
|
light-pass) printf '%s' "$PASS_MODEL" ;;
|
|
write-unavailable) printf '%s' "$PASS_MODEL" ;;
|
|
repair) printf '%s' "$REPAIR_MODEL" ;;
|
|
timeout-cancel) printf '%s' "$SLOW_MODEL" ;;
|
|
*) die "unknown scenario: $1" ;;
|
|
esac
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Input parsing and validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
usage: e2e-hot-path-agents.sh --self-test
|
|
e2e-hot-path-agents.sh --preflight-only --claude <path> --pi <path>
|
|
--runtime-evidence <file> --base-url <url>
|
|
--direct-model <alias> --pass-model <alias>
|
|
--repair-model <alias> --slow-model <alias>
|
|
--edge-bin <path> --edge-config <file> --pi-config-dir <dir>
|
|
--pi-provider <name> --observation-file <file>
|
|
--workspace-root <dir> --output <file>
|
|
--claude-secret-env <name> --pi-secret-env <name>
|
|
[--fixture <file>]
|
|
e2e-hot-path-agents.sh --run (same inputs as --preflight-only)
|
|
EOF
|
|
}
|
|
|
|
parse_run_inputs() {
|
|
CLAUDE_BIN=""
|
|
PI_BIN=""
|
|
RUNTIME_EVIDENCE=""
|
|
FIXTURE_PATH="$SCHEMA_PATH"
|
|
BASE_URL=""
|
|
DIRECT_MODEL=""
|
|
PASS_MODEL=""
|
|
REPAIR_MODEL=""
|
|
SLOW_MODEL=""
|
|
EDGE_BIN=""
|
|
EDGE_CONFIG=""
|
|
PI_CONFIG_DIR=""
|
|
PI_PROVIDER=""
|
|
OBSERVATION_FILE=""
|
|
WORKSPACE_ROOT=""
|
|
OUTPUT_PATH=""
|
|
CLAUDE_SECRET_ENV=""
|
|
PI_SECRET_ENV=""
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--claude) CLAUDE_BIN="${2:-}"; shift 2 ;;
|
|
--pi) PI_BIN="${2:-}"; shift 2 ;;
|
|
--runtime-evidence) RUNTIME_EVIDENCE="${2:-}"; shift 2 ;;
|
|
--fixture) FIXTURE_PATH="${2:-}"; shift 2 ;;
|
|
--base-url) BASE_URL="${2:-}"; shift 2 ;;
|
|
--direct-model) DIRECT_MODEL="${2:-}"; shift 2 ;;
|
|
--pass-model) PASS_MODEL="${2:-}"; shift 2 ;;
|
|
--repair-model) REPAIR_MODEL="${2:-}"; shift 2 ;;
|
|
--slow-model) SLOW_MODEL="${2:-}"; shift 2 ;;
|
|
--edge-bin) EDGE_BIN="${2:-}"; shift 2 ;;
|
|
--edge-config) EDGE_CONFIG="${2:-}"; shift 2 ;;
|
|
--pi-config-dir) PI_CONFIG_DIR="${2:-}"; shift 2 ;;
|
|
--pi-provider) PI_PROVIDER="${2:-}"; shift 2 ;;
|
|
--observation-file) OBSERVATION_FILE="${2:-}"; shift 2 ;;
|
|
--workspace-root) WORKSPACE_ROOT="${2:-}"; shift 2 ;;
|
|
--output) OUTPUT_PATH="${2:-}"; shift 2 ;;
|
|
--claude-secret-env) CLAUDE_SECRET_ENV="${2:-}"; shift 2 ;;
|
|
--pi-secret-env) PI_SECRET_ENV="${2:-}"; shift 2 ;;
|
|
*) die_usage "unknown option: $1" ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
validate_inputs_presence() {
|
|
[ -n "$CLAUDE_BIN" ] || die_validation "missing --claude binary"
|
|
[ -n "$PI_BIN" ] || die_validation "missing --pi binary"
|
|
[ -x "$CLAUDE_BIN" ] || die_validation "claude binary not executable"
|
|
[ -x "$PI_BIN" ] || die_validation "pi binary not executable"
|
|
[ -n "$RUNTIME_EVIDENCE" ] || die_validation "missing --runtime-evidence"
|
|
[ -f "$RUNTIME_EVIDENCE" ] || die_validation "runtime-evidence file absent"
|
|
[ -n "$FIXTURE_PATH" ] || die_validation "missing --fixture"
|
|
[ -f "$FIXTURE_PATH" ] || die_validation "fixture file absent"
|
|
[ -n "$BASE_URL" ] || die_validation "missing --base-url"
|
|
[ -n "$DIRECT_MODEL" ] || die_validation "missing --direct-model alias"
|
|
[ -n "$PASS_MODEL" ] || die_validation "missing --pass-model alias"
|
|
[ -n "$REPAIR_MODEL" ] || die_validation "missing --repair-model alias"
|
|
[ -n "$SLOW_MODEL" ] || die_validation "missing --slow-model alias"
|
|
[ -n "$EDGE_BIN" ] || die_validation "missing --edge-bin"
|
|
[ -x "$EDGE_BIN" ] || die_validation "edge binary not executable"
|
|
[ -n "$EDGE_CONFIG" ] || die_validation "missing --edge-config"
|
|
[ -f "$EDGE_CONFIG" ] || die_validation "edge-config file absent"
|
|
[ -n "$PI_CONFIG_DIR" ] || die_validation "missing --pi-config-dir"
|
|
[ -d "$PI_CONFIG_DIR" ] || die_validation "pi-config-dir absent"
|
|
[ -n "$PI_PROVIDER" ] || die_validation "missing --pi-provider"
|
|
[ -n "$OBSERVATION_FILE" ] || die_validation "missing --observation-file"
|
|
[ -f "$OBSERVATION_FILE" ] || die_validation "observation-file absent"
|
|
[ -n "$WORKSPACE_ROOT" ] || die_validation "missing --workspace-root"
|
|
[ -d "$WORKSPACE_ROOT" ] || die_validation "workspace-root absent"
|
|
[ -n "$OUTPUT_PATH" ] || die_validation "missing --output"
|
|
[ -n "$CLAUDE_SECRET_ENV" ] || die_validation "missing --claude-secret-env"
|
|
[ -n "$PI_SECRET_ENV" ] || die_validation "missing --pi-secret-env"
|
|
# Presence-only secret check: the named env vars must be set and non-empty.
|
|
# Values are never read or printed.
|
|
[ -n "${!CLAUDE_SECRET_ENV:-}" ] || die_validation "claude secret env not present"
|
|
[ -n "${!PI_SECRET_ENV:-}" ] || die_validation "pi secret env not present"
|
|
return 0
|
|
}
|
|
|
|
# Compare a caller-supplied evidence field to an actual computed value without
|
|
# ever echoing either value (only the field name appears on mismatch).
|
|
assert_digest_matches() {
|
|
local actual="$1" supplied_file="$2" field="$3"
|
|
local supplied
|
|
supplied=$(jq -r --arg f "$field" '.[$f] // empty' "$supplied_file" 2>/dev/null) \
|
|
|| die_validation "$field: evidence file is not valid JSON"
|
|
[ -n "$supplied" ] || die_validation "$field: missing from evidence"
|
|
[ "$supplied" = "$actual" ] || die_validation "$field: identity mismatch"
|
|
}
|
|
|
|
# Validate current source/worktree identity against the runtime evidence. Sets
|
|
# SOURCE_HEAD/SOURCE_TREE for the manifest.
|
|
validate_worktree_fingerprint() {
|
|
local actual_script actual_schema actual_fp
|
|
actual_script=$(sha256_file "$SELF_PATH")
|
|
actual_schema=$(sha256_file "$SCHEMA_PATH")
|
|
assert_digest_matches "$actual_script" "$RUNTIME_EVIDENCE" "script_sha256"
|
|
assert_digest_matches "$actual_schema" "$RUNTIME_EVIDENCE" "schema_sha256"
|
|
SOURCE_HEAD=$(git_head)
|
|
SOURCE_TREE=$(git_tree)
|
|
assert_digest_matches "$SOURCE_HEAD" "$RUNTIME_EVIDENCE" "head"
|
|
assert_digest_matches "$SOURCE_TREE" "$RUNTIME_EVIDENCE" "source_tree"
|
|
actual_fp=$(worktree_fingerprint)
|
|
assert_digest_matches "$actual_fp" "$RUNTIME_EVIDENCE" "worktree_fingerprint"
|
|
}
|
|
|
|
# Validate the selected Edge binary/config, Pi config dir, CLI binaries and the
|
|
# fixture against the runtime evidence. Sets manifest digest globals.
|
|
validate_edge_binary_config_fixture_identity() {
|
|
local actual_claude actual_pi actual_edge actual_edge_cfg actual_pi_cfg actual_fixture
|
|
actual_claude=$(sha256_file "$CLAUDE_BIN")
|
|
actual_pi=$(sha256_file "$PI_BIN")
|
|
actual_edge=$(sha256_file "$EDGE_BIN")
|
|
actual_edge_cfg=$(sha256_file "$EDGE_CONFIG")
|
|
actual_pi_cfg=$(tree_sha256 "$PI_CONFIG_DIR")
|
|
actual_fixture=$(sha256_file "$FIXTURE_PATH")
|
|
assert_digest_matches "$actual_claude" "$RUNTIME_EVIDENCE" "claude_binary_sha256"
|
|
assert_digest_matches "$actual_pi" "$RUNTIME_EVIDENCE" "pi_binary_sha256"
|
|
assert_digest_matches "$actual_edge" "$RUNTIME_EVIDENCE" "edge_binary_sha256"
|
|
assert_digest_matches "$actual_edge_cfg" "$RUNTIME_EVIDENCE" "edge_config_sha256"
|
|
assert_digest_matches "$actual_pi_cfg" "$RUNTIME_EVIDENCE" "pi_config_sha256"
|
|
assert_digest_matches "$actual_fixture" "$RUNTIME_EVIDENCE" "fixture_sha256"
|
|
RUNTIME_SHA256=$(sha256_file "$RUNTIME_EVIDENCE")
|
|
FIXTURE_SHA256="$actual_fixture"
|
|
CLAUDE_BIN_SHA256="$actual_claude"
|
|
PI_BIN_SHA256="$actual_pi"
|
|
}
|
|
|
|
# Validate the base/profile identity and the four scenario preset aliases against
|
|
# the runtime evidence by digest only. Endpoint and model values are never
|
|
# printed or serialized.
|
|
validate_runner_and_profile_identity() {
|
|
assert_digest_matches "$(sha256_str "$BASE_URL")" "$RUNTIME_EVIDENCE" "base_url_sha256"
|
|
assert_digest_matches "$(sha256_str "$PI_PROVIDER")" "$RUNTIME_EVIDENCE" "pi_provider_sha256"
|
|
assert_digest_matches "$(sha256_str "$DIRECT_MODEL")" "$RUNTIME_EVIDENCE" "direct_model_sha256"
|
|
assert_digest_matches "$(sha256_str "$PASS_MODEL")" "$RUNTIME_EVIDENCE" "pass_model_sha256"
|
|
assert_digest_matches "$(sha256_str "$REPAIR_MODEL")" "$RUNTIME_EVIDENCE" "repair_model_sha256"
|
|
assert_digest_matches "$(sha256_str "$SLOW_MODEL")" "$RUNTIME_EVIDENCE" "slow_model_sha256"
|
|
}
|
|
|
|
# The observation log must be a readable regular file (a live Edge log holding
|
|
# JSON hot_path_observation records). Per-case freshness is enforced during the
|
|
# run, not here.
|
|
validate_observation_log_preflight() {
|
|
[ -f "$OBSERVATION_FILE" ] || die_validation "observation-file is not a regular file"
|
|
[ -r "$OBSERVATION_FILE" ] || die_validation "observation-file not readable"
|
|
log "observation log preflight ok"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario fixtures and agent invocation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
scenario_prompt() {
|
|
case "$1" in
|
|
direct) printf 'Summarize the workspace README in one short line.' ;;
|
|
light-pass) printf 'Author the plan/review pair and complete the task.' ;;
|
|
repair) printf 'The seeded file has a defect; author plan/review, fix and verify.' ;;
|
|
write-unavailable) printf 'Author the plan/review pair under the job directory.' ;;
|
|
timeout-cancel) printf 'Perform a long running analysis of the workspace.' ;;
|
|
*) die "unknown scenario: $1" ;;
|
|
esac
|
|
}
|
|
|
|
# Validate one production Hot Path lifecycle and collapse retry attempts into the
|
|
# manifest's one-row-per-stage projection. Return 2 while the lifecycle is still
|
|
# open and 1 for a closed contradiction or malformed production record.
|
|
reduce_observation_fragment() {
|
|
local scenario="$1" frag="$2" projected
|
|
projected=$(jq -c -s '
|
|
. as $all
|
|
| if any($all[];
|
|
type == "object"
|
|
and (.msg // "") != "hot_path_observation"
|
|
and (has("hot_path_event_class") or has("hot_path_request_id")))
|
|
then error("foreign hot path observation lookalike")
|
|
else
|
|
[ $all[]
|
|
| select(type == "object" and .msg == "hot_path_observation")
|
|
| {raw_rid:(.hot_path_request_id // ""),
|
|
ec:(.hot_path_event_class // ""),
|
|
sk:(.hot_path_stage_kind // ""),
|
|
attempt:(.hot_path_attempt_bucket // ""),
|
|
disposition:(.hot_path_disposition // ""),
|
|
reason:(.hot_path_reason // ""),
|
|
cleanup:(.hot_path_cleanup_outcome // ""),
|
|
orphan:(.hot_path_orphan_outcome // "")}
|
|
]
|
|
end
|
|
' "$frag" 2>/dev/null) || return 1
|
|
|
|
local record_count rid_count
|
|
record_count=$(jq 'length' <<<"$projected") || return 1
|
|
[ "$record_count" -gt 0 ] || return 2
|
|
jq -e '
|
|
def oneof($xs): . as $v | any($xs[]; . == $v);
|
|
all(.[];
|
|
(.raw_rid | type == "string" and length > 0)
|
|
and (.ec | oneof(["dispatch","stage","light","terminal","cleanup","orphan"]))
|
|
and (.sk | oneof(["","selector","local","review","cleanup"]))
|
|
and (.attempt | oneof(["","first","retry"]))
|
|
and (.disposition | oneof(["","success","tool_turn","length","provider_error","validation_error","timeout","caller_cancel"]))
|
|
and (.reason | oneof(["","mode_disabled","artifact_required","invalid_input","provider_error","timeout","caller_cancel"]))
|
|
and (.cleanup | oneof(["","success","primary_error","ttl_expired"]))
|
|
and (.orphan | oneof(["","ttl_expired","cleanup_failed"]))
|
|
and (if .ec == "dispatch" then
|
|
.sk == "" and .attempt == "" and .disposition == "" and .cleanup == "" and .orphan == ""
|
|
elif .ec == "stage" then
|
|
(.sk | IN("local","review")) and (.attempt | IN("first","retry"))
|
|
and (.disposition != "") and .reason == "" and .cleanup == "" and .orphan == ""
|
|
elif .ec == "light" then
|
|
(.sk | IN("review","cleanup")) and (.attempt | IN("first","retry"))
|
|
and .disposition == "" and .reason == "" and .cleanup == "" and .orphan == ""
|
|
elif .ec == "terminal" then
|
|
.sk == "" and .attempt == "" and .disposition != ""
|
|
and .reason == "" and .cleanup == "" and .orphan == ""
|
|
elif .ec == "cleanup" then
|
|
.sk == "" and .attempt == "" and .disposition == ""
|
|
and .reason == "" and .cleanup != "" and .orphan == ""
|
|
else
|
|
.sk == "" and .attempt == "" and .disposition == ""
|
|
and .reason == "" and .cleanup == "" and .orphan != ""
|
|
end)
|
|
)
|
|
' <<<"$projected" >/dev/null 2>&1 || return 1
|
|
rid_count=$(jq '[.[].raw_rid] | unique | length' <<<"$projected") || return 1
|
|
[ "$rid_count" -eq 1 ] || return 1
|
|
|
|
local closure_count
|
|
if [ "$scenario" = timeout-cancel ]; then
|
|
closure_count=$(jq '[.[] | select(.ec == "stage" and .sk == "local" and (.disposition | IN("caller_cancel","timeout")))] | length' <<<"$projected") || return 1
|
|
elif [ "$scenario" = write-unavailable ]; then
|
|
closure_count=$(jq '[.[] | select(.ec == "dispatch" and .reason != "")] | length' <<<"$projected") || return 1
|
|
else
|
|
closure_count=$(jq '[.[] | select(.ec == "terminal")] | length' <<<"$projected") || return 1
|
|
fi
|
|
[ "$closure_count" -gt 0 ] || return 2
|
|
|
|
# The production lifecycle is closed by a terminal for admitted direct/light
|
|
# cases, by the bounded rejection reason for failed admission, and by the
|
|
# immediate local caller-cancel stage for harness-owned child cancellation.
|
|
# Stage attempts may repeat, but only a terminal success can close each
|
|
# successful stage.
|
|
jq -e --arg scenario "$scenario" '
|
|
def stage_rows($kind):
|
|
[to_entries[] | select(.value.ec == "stage" and .value.sk == $kind)];
|
|
def light_rows($kind):
|
|
[to_entries[] | select(.value.ec == "light" and .value.sk == $kind)];
|
|
def attempts_close($rows):
|
|
($rows | length) > 0
|
|
and $rows[0].value.attempt == "first"
|
|
and all($rows[1:][]; .value.attempt == "retry")
|
|
and all($rows[0:-1][]; .value.disposition == "tool_turn")
|
|
and $rows[-1].value.disposition == "success";
|
|
. as $p
|
|
| if $scenario == "direct" then
|
|
($p | length) == 2
|
|
and $p[0].ec == "dispatch" and $p[0].reason == ""
|
|
and $p[1].ec == "terminal" and $p[1].disposition == "success"
|
|
elif $scenario == "write-unavailable" then
|
|
($p | length) == 1
|
|
and $p[0].ec == "dispatch" and $p[0].reason != ""
|
|
elif ($scenario == "light-pass" or $scenario == "repair") then
|
|
stage_rows("local") as $local
|
|
| stage_rows("review") as $review
|
|
| light_rows("review") as $review_transition
|
|
| light_rows("cleanup") as $cleanup_transition
|
|
| [to_entries[] | select(.value.ec == "cleanup")] as $cleanup
|
|
| [to_entries[] | select(.value.ec == "terminal")] as $terminal
|
|
| [to_entries[] | select(.value.ec == "dispatch")] as $dispatch
|
|
| [to_entries[] | select(.value.ec == "orphan")] as $orphan
|
|
| ($dispatch | length) == 1 and $dispatch[0].key == 0 and $dispatch[0].value.reason == ""
|
|
and attempts_close($local) and attempts_close($review)
|
|
and ($review_transition | length) == (if $scenario == "repair" then 2 else 1 end)
|
|
and $review_transition[0].value.attempt == "first"
|
|
and all($review_transition[1:][]; .value.attempt == "retry")
|
|
and ($cleanup_transition | length) == 1 and $cleanup_transition[0].value.attempt == "first"
|
|
and ($cleanup | length) == 1 and $cleanup[0].value.cleanup == "success"
|
|
and ($terminal | length) == 1 and $terminal[0].value.disposition == "success"
|
|
and ($orphan | length) == 0
|
|
and $local[0].key == 1
|
|
and $local[-1].key < $review_transition[0].key
|
|
and $review_transition[0].key < $review[0].key
|
|
and $review[-1].key < $cleanup_transition[0].key
|
|
and $cleanup_transition[0].key + 1 == $cleanup[0].key
|
|
and $cleanup[0].key + 1 == $terminal[0].key
|
|
and $terminal[0].key + 1 == ($p | length)
|
|
and ($p | length) == (1 + ($local|length) + ($review|length)
|
|
+ ($review_transition|length) + 1 + 1 + 1)
|
|
else
|
|
stage_rows("local") as $local
|
|
| [to_entries[] | select(.value.ec == "dispatch")] as $dispatch
|
|
| [to_entries[] | select(.value.ec == "orphan")] as $orphan
|
|
| [to_entries[] | select(.value.ec == "terminal" or .value.ec == "cleanup" or .value.ec == "light" or (.value.ec == "stage" and .value.sk != "local"))] as $foreign
|
|
| ($dispatch | length) == 1 and $dispatch[0].key == 0 and $dispatch[0].value.reason == ""
|
|
and ($local | length) > 0 and $local[0].key == 1 and $local[0].value.attempt == "first"
|
|
and all($local[1:][]; .value.attempt == "retry")
|
|
and all($local[0:-1][]; .value.disposition == "tool_turn")
|
|
and ($local[-1].value.disposition | IN("caller_cancel","timeout"))
|
|
and ($orphan | length) == 0
|
|
and $local[-1].key + 1 == ($p | length)
|
|
and ($foreign | length) == 0
|
|
and ($p | length) == (1 + ($local|length))
|
|
end
|
|
' <<<"$projected" >/dev/null 2>&1 || return 1
|
|
|
|
local raw_rid proj_rid
|
|
raw_rid=$(jq -r '.[0].raw_rid' <<<"$projected") || return 1
|
|
proj_rid="rid-$(sha256_str "$raw_rid" | sed 's/^sha256://' | cut -c1-8)"
|
|
jq -c --arg rid "$proj_rid" --arg scenario "$scenario" '
|
|
if $scenario == "direct" then
|
|
[{request_id:$rid,stage:"selector",outcome:"observed"}]
|
|
elif $scenario == "write-unavailable" then
|
|
[{request_id:$rid,stage:"selector",outcome:"failed"}]
|
|
elif $scenario == "timeout-cancel" then
|
|
[{request_id:$rid,stage:"selector",outcome:"observed"},
|
|
{request_id:$rid,stage:"local",outcome:"observed"}]
|
|
else
|
|
[{request_id:$rid,stage:"selector",outcome:"observed"},
|
|
{request_id:$rid,stage:"local",outcome:"observed"},
|
|
{request_id:$rid,stage:"review",outcome:"observed"},
|
|
{request_id:$rid,stage:"cleanup",outcome:"observed"}]
|
|
end
|
|
' <<<"$projected"
|
|
}
|
|
|
|
# Read the observation records appended by the selected runtime to the live log
|
|
# after the case started. Poll until a scenario-specific closure is stable for a
|
|
# short quiet interval, bounded by the configured lifecycle deadline.
|
|
capture_appended_observation() {
|
|
local case_id="$1" scenario="$2" offset_before="$3" inode_before="$4"
|
|
local f="$OBSERVATION_FILE"
|
|
[ -f "$f" ] || return 1
|
|
local frag="$RAW_CAPTURE_DIR/obs-appended-${case_id}"
|
|
local wait_msec="$OBSERVATION_WAIT_MSEC"
|
|
[ "$scenario" = timeout-cancel ] && wait_msec="$OBSERVATION_CANCEL_WAIT_MSEC"
|
|
local start_ms now_ms deadline_ms cur_inode cur_size last_closed_size=-1 closed_since=0
|
|
local candidate rc
|
|
start_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || printf '0')
|
|
deadline_ms=$((start_ms + wait_msec))
|
|
while :; do
|
|
[ -f "$f" ] || return 1
|
|
cur_inode=$(stat -c '%i' "$f" 2>/dev/null || printf '0')
|
|
cur_size=$(stat -c '%s' "$f" 2>/dev/null || printf '0')
|
|
[ "$cur_inode" = "$inode_before" ] || return 1
|
|
[ "$cur_size" -ge "$offset_before" ] || return 1
|
|
tail -c "+$((offset_before + 1))" "$f" > "$frag" 2>/dev/null || return 1
|
|
if candidate=$(reduce_observation_fragment "$scenario" "$frag"); then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
[ "$rc" -ne 1 ] || return 1
|
|
now_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || printf '0')
|
|
if [ "$rc" -eq 0 ]; then
|
|
if [ "$cur_size" -ne "$last_closed_size" ]; then
|
|
last_closed_size="$cur_size"
|
|
closed_since="$now_ms"
|
|
elif [ $((now_ms - closed_since)) -ge "$OBSERVATION_QUIET_MSEC" ]; then
|
|
printf '%s' "$candidate"
|
|
return 0
|
|
fi
|
|
else
|
|
last_closed_size=-1
|
|
closed_since=0
|
|
fi
|
|
[ "$now_ms" -lt "$deadline_ms" ] || return 1
|
|
sleep 0.05
|
|
done
|
|
}
|
|
|
|
workspace_snapshot() {
|
|
local ws="$1"
|
|
local artifacts=false
|
|
if [ -e "$ws/.iop/job" ] && [ -n "$(find "$ws/.iop/job" -mindepth 1 -print -quit 2>/dev/null)" ]; then
|
|
artifacts=true
|
|
fi
|
|
local writable=false mode
|
|
mode=$(stat -c '%A' "$ws")
|
|
if [[ "${mode:2:1}${mode:5:1}${mode:8:1}" == *w* ]]; then writable=true; fi
|
|
printf '{"artifacts_present":%s,"writable":%s,"tree_sha256":"%s"}' \
|
|
"$artifacts" "$writable" "$(tree_sha256 "$ws")"
|
|
}
|
|
|
|
# Parse captured agent stdout (JSONL) into visible_event summaries. The agent
|
|
# field selects the native shape. Raw content is never emitted; only safe kinds
|
|
# and short labels are recorded. A single jq pass parses the whole stream so the
|
|
# per-event subprocess pipeline cost (catastrophic on slow filesystems) is
|
|
# avoided and the visible_event index stays deterministically sequential.
|
|
parse_visible_events() {
|
|
local agent="$1" out_file="$2" child_status="${3:-0}"
|
|
local triggered="${4:-false}" target="${5:-none}" events
|
|
events=$(jq -c -s --arg agent "$agent" '
|
|
def tool_detail($name; $args):
|
|
($name // "" | ascii_downcase) as $n
|
|
| ($args // {} | tojson | ascii_downcase) as $a
|
|
| if ($n | test("cleanup|delete|remove"))
|
|
or (($a | test("\\.iop/job")) and ($a | test("rm |delete|remove")))
|
|
then "workspace_cleanup"
|
|
elif ($n | test("repair")) or ($a | test("seeded\\.txt|repair")) then "repair_write"
|
|
elif ($n | test("review")) or ($a | test("review\\.md")) then "review_write"
|
|
elif ($n | test("write|plan")) or ($a | test("plan\\.md|\\.iop/job")) then "workspace_write"
|
|
else "tool_call" end;
|
|
if $agent == "claude" then
|
|
[ .[]
|
|
| if .type == "system" then {kind:"system_init", detail:"init"}
|
|
elif .type == "assistant" then
|
|
(.message.content // [])[]
|
|
| if .type == "tool_use"
|
|
then {kind:"tool_use", detail:tool_detail(.name; .input)}
|
|
else {kind:"assistant_text", detail:"text"} end
|
|
elif .type == "user" then
|
|
(.message.content // [])[]
|
|
| if .type == "tool_result" then
|
|
if (.is_error // false) then {kind:"tool_result", detail:"error"}
|
|
else {kind:"tool_result", detail:"ok"} end
|
|
else {kind:"partial", detail:"event"} end
|
|
elif .type == "result" then
|
|
if .subtype == "success" then {kind:"terminal_success", detail:"success"}
|
|
elif (.subtype | IN("cancelled","canceled","interrupted"))
|
|
then {kind:"terminal_cancelled", detail:"cancelled"}
|
|
else {kind:"terminal_error", detail:"provider_error"} end
|
|
else empty end
|
|
]
|
|
else
|
|
reduce .[] as $e
|
|
({visible:[], final_assistant:null, agent_end_count:0, invalid:false};
|
|
if $e.type == "agent_start" then
|
|
.visible += [{kind:"system_init",detail:"init"}]
|
|
elif $e.type == "message_update" and $e.message.role == "assistant" then
|
|
.visible += [{kind:"partial",detail:"delta"}]
|
|
elif $e.type == "message_end" and $e.message.role == "assistant" then
|
|
.final_assistant = $e.message
|
|
| if any($e.message.content[]?; .type == "text" or .type == "thinking")
|
|
then .visible += [{kind:"assistant_text",detail:"text"}]
|
|
else . end
|
|
elif $e.type == "tool_execution_start" then
|
|
.visible += [{kind:"tool_use",detail:tool_detail($e.toolName;$e.args)}]
|
|
elif $e.type == "tool_execution_end" then
|
|
.visible += [{kind:"tool_result",detail:(if ($e.isError // false) then "error" else "ok" end)}]
|
|
elif $e.type == "agent_end" then
|
|
.agent_end_count += 1
|
|
| (([$e.messages[]? | select(.role == "assistant")] | last) // .final_assistant) as $final
|
|
| if $final == null then .invalid = true
|
|
elif $final.stopReason == "stop" then
|
|
.visible += [{kind:"terminal_success",detail:"success"}]
|
|
elif ($final.stopReason | IN("error","aborted","length","toolUse")) then
|
|
.visible += [{kind:"terminal_error",detail:"provider_error"}]
|
|
else .invalid = true end
|
|
else . end)
|
|
| if .invalid or .agent_end_count > 1 then error("invalid Pi lifecycle")
|
|
else .visible end
|
|
end
|
|
| to_entries
|
|
| map({index:.key, kind:.value.kind, detail:.value.detail})
|
|
' "$out_file" 2>/dev/null) || return 1
|
|
|
|
# Pi 0.81.1 disposes and exits 143 on SIGTERM without an AgentSessionEvent
|
|
# terminal. Only the harness-owned child-only signal may close that exact
|
|
# process state as cancellation, and never over a contradictory terminal.
|
|
if [ "$agent" = pi ] && [ "$child_status" -eq 143 ] \
|
|
&& [ "$triggered" = true ] && [ "$target" = child_only ]; then
|
|
local terminal_count next_index
|
|
terminal_count=$(jq '[.[] | select(.kind | startswith("terminal_"))] | length' <<<"$events") || return 1
|
|
if [ "$terminal_count" -eq 0 ]; then
|
|
next_index=$(jq 'length' <<<"$events") || return 1
|
|
events=$(jq -c --argjson i "$next_index" \
|
|
'. + [{index:$i,kind:"terminal_cancelled",detail:"cancelled"}]' <<<"$events") || return 1
|
|
fi
|
|
fi
|
|
printf '%s' "$events"
|
|
}
|
|
|
|
# Derive the public result only from correlated process, protocol, observation,
|
|
# cancellation, and workspace facts. Scenario names select invariants; they are
|
|
# never copied into outcome/terminal/cleanup without these checks succeeding.
|
|
derive_case_result() {
|
|
local agent="$1" scenario="$2" child_status="$3" triggered="$4" target="$5"
|
|
local sentinel_survived="$6" visible_events="$7" observation="$8"
|
|
local snapshot_before="$9" snapshot_after="${10}"
|
|
local terminal_kind terminal_count outcome terminal cleanup
|
|
|
|
terminal_count=$(jq '[.[] | select(.kind | startswith("terminal_"))] | length' <<<"$visible_events")
|
|
[ "$terminal_count" -eq 1 ] || return 1
|
|
jq -e 'length > 0 and (.[-1].kind | startswith("terminal_"))' \
|
|
<<<"$visible_events" >/dev/null || return 1
|
|
terminal_kind=$(jq -r '.[-1].kind' <<<"$visible_events")
|
|
case "$terminal_kind" in
|
|
terminal_success)
|
|
[ "$child_status" -eq 0 ] && [ "$triggered" = false ] && [ "$target" = none ] || return 1
|
|
outcome=completed; terminal=success
|
|
;;
|
|
terminal_error)
|
|
case "$agent" in
|
|
pi) [ "$child_status" -eq 0 ] ;;
|
|
claude) [ "$child_status" -ne 0 ] ;;
|
|
*) return 1 ;;
|
|
esac
|
|
[ "$triggered" = false ] && [ "$target" = none ] || return 1
|
|
outcome=error; terminal=provider_error
|
|
;;
|
|
terminal_cancelled)
|
|
[ "$child_status" -ne 0 ] && [ "$triggered" = true ] \
|
|
&& [ "$target" = child_only ] && [ "$sentinel_survived" = true ] || return 1
|
|
outcome=cancelled; terminal=cancelled
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
[ "$sentinel_survived" = true ] || return 1
|
|
|
|
# A terminal-only stream is not evidence that the agent exposed the Hot Path
|
|
# work. Require the scenario's visible tool progression in causal order.
|
|
case "$scenario" in
|
|
direct)
|
|
jq -e 'any(.[]; .kind == "assistant_text" or .kind == "partial")' \
|
|
<<<"$visible_events" >/dev/null || return 1
|
|
;;
|
|
light-pass)
|
|
jq -e '
|
|
[.[] | select(.kind == "tool_use") | .detail] as $t
|
|
| ($t | index("workspace_write")) as $write
|
|
| ($t | index("review_write")) as $review
|
|
| ($t | index("workspace_cleanup")) as $cleanup
|
|
| $write != null and $review != null and $cleanup != null
|
|
and $write < $review and $review < $cleanup
|
|
' <<<"$visible_events" >/dev/null || return 1
|
|
;;
|
|
repair)
|
|
jq -e '
|
|
[.[] | select(.kind == "tool_use") | .detail] as $t
|
|
| ($t | index("workspace_write")) as $write
|
|
| ($t | index("review_write")) as $review
|
|
| ($t | index("repair_write")) as $repair
|
|
| ($t | index("workspace_cleanup")) as $cleanup
|
|
| $write != null and $review != null and $repair != null and $cleanup != null
|
|
and $write < $review and $review < $repair and $repair < $cleanup
|
|
' <<<"$visible_events" >/dev/null || return 1
|
|
;;
|
|
write-unavailable)
|
|
jq -e '
|
|
any(.[]; .kind == "tool_use" and .detail == "workspace_write")
|
|
and any(.[]; .kind == "tool_result" and .detail == "error")
|
|
' <<<"$visible_events" >/dev/null || return 1
|
|
;;
|
|
timeout-cancel)
|
|
jq -e 'any(.[]; .kind == "tool_use" and .detail == "workspace_write")' \
|
|
<<<"$visible_events" >/dev/null || return 1
|
|
;;
|
|
esac
|
|
|
|
if [ "$terminal" = cancelled ] \
|
|
&& jq -e '.artifacts_present == true' <<<"$snapshot_after" >/dev/null; then
|
|
cleanup=orphan
|
|
elif jq -e 'any(.[]; .stage == "cleanup" and .outcome == "observed")' \
|
|
<<<"$observation" >/dev/null \
|
|
&& jq -e '.artifacts_present == false' <<<"$snapshot_after" >/dev/null; then
|
|
cleanup=removed
|
|
else
|
|
cleanup=none
|
|
fi
|
|
|
|
case "$scenario" in
|
|
direct)
|
|
[ "$outcome:$terminal:$cleanup" = "completed:success:none" ] || return 1
|
|
jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" '
|
|
($b.artifacts_present == false) and ($a.artifacts_present == false)
|
|
and ($b.writable == true) and ($a.writable == true)
|
|
and ($b.tree_sha256 == $a.tree_sha256)
|
|
' -n >/dev/null || return 1
|
|
;;
|
|
light-pass|repair)
|
|
[ "$outcome:$terminal:$cleanup" = "completed:success:removed" ] || return 1
|
|
jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" '
|
|
($b.artifacts_present == false) and ($a.artifacts_present == false)
|
|
and ($b.writable == true) and ($a.writable == true)
|
|
and ($b.tree_sha256 != $a.tree_sha256)
|
|
' -n >/dev/null || return 1
|
|
;;
|
|
write-unavailable)
|
|
[ "$outcome:$terminal:$cleanup" = "error:provider_error:none" ] || return 1
|
|
jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" '
|
|
($b.artifacts_present == false) and ($a.artifacts_present == false)
|
|
and ($b.writable == false) and ($a.writable == false)
|
|
and ($b.tree_sha256 == $a.tree_sha256)
|
|
' -n >/dev/null || return 1
|
|
;;
|
|
timeout-cancel)
|
|
[ "$outcome:$terminal:$cleanup" = "cancelled:cancelled:orphan" ] || return 1
|
|
jq -e --argjson b "$snapshot_before" --argjson a "$snapshot_after" '
|
|
($b.artifacts_present == false) and ($a.artifacts_present == true)
|
|
and ($b.writable == true) and ($a.writable == true)
|
|
and ($b.tree_sha256 != $a.tree_sha256)
|
|
' -n >/dev/null || return 1
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
printf '%s:%s:%s' "$outcome" "$terminal" "$cleanup"
|
|
}
|
|
|
|
# Run a single matrix case. Produces a case evidence JSON object on stdout.
|
|
run_case() {
|
|
local agent="$1" scenario="$2" case_id="$agent:$scenario" request_id
|
|
request_id=$(request_id_for "$case_id")
|
|
local ws="$WORKSPACE_ROOT/$case_id"
|
|
rm -rf "$ws"
|
|
mkdir -p "$ws"
|
|
if [ "$scenario" = repair ]; then printf 'defect marker\n' > "$ws/seeded.txt"; fi
|
|
if [ "$scenario" = write-unavailable ]; then
|
|
chmod a-w "$ws" || return 1
|
|
fi
|
|
local snapshot_before
|
|
snapshot_before=$(workspace_snapshot "$ws")
|
|
|
|
local prompt model provider agent_bin
|
|
prompt=$(scenario_prompt "$scenario")
|
|
model=$(scenario_model_alias "$scenario")
|
|
if [ "$agent" = claude ]; then
|
|
provider="claude"; agent_bin="$CLAUDE_BIN"
|
|
else
|
|
provider="$PI_PROVIDER"; agent_bin="$PI_BIN"
|
|
fi
|
|
|
|
local argv_file="$RAW_CAPTURE_DIR/argv-${case_id}.expected"
|
|
local recorded_file="$RAW_CAPTURE_DIR/argv-${case_id}.recorded"
|
|
local out_file="$RAW_CAPTURE_DIR/out-${case_id}.jsonl"
|
|
local err_file="$RAW_CAPTURE_DIR/err-${case_id}.log"
|
|
if [ "$agent" = claude ]; then build_claude_argv "$prompt" > "$argv_file"
|
|
else build_pi_argv "$provider" "$model" "$prompt" > "$argv_file"; fi
|
|
local argv_hash
|
|
argv_hash=$(sha256_file "$argv_file")
|
|
: > "$out_file"; : > "$err_file"
|
|
|
|
local -a argv_arr=()
|
|
local tok
|
|
while IFS= read -r -d '' tok; do argv_arr+=("$tok"); done < "$argv_file"
|
|
|
|
# Snapshot the observation-log identity and byte offset immediately before
|
|
# invocation so only records appended by this case are consumed afterward.
|
|
local obs_offset_before obs_inode_before
|
|
obs_offset_before=$(stat -c '%s' "$OBSERVATION_FILE" 2>/dev/null || printf '0')
|
|
obs_inode_before=$(stat -c '%i' "$OBSERVATION_FILE" 2>/dev/null || printf '0')
|
|
|
|
local sentinel_pid child_pid
|
|
sleep "$SHARED_SENTINEL_LIFE_SEC" >/dev/null 2>&1 & sentinel_pid=$!
|
|
local triggered=false target=none start_ms end_ms child_status=0
|
|
start_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || echo 0)
|
|
(
|
|
cd "$ws" || exit 1
|
|
ANTHROPIC_BASE_URL="$BASE_URL" \
|
|
ANTHROPIC_MODEL="$model" \
|
|
PI_CODING_AGENT_DIR="$PI_CONFIG_DIR" \
|
|
IOP_HOT_PATH_FAKE_AGENT="$agent" \
|
|
IOP_HOT_PATH_FAKE_SCENARIO="$scenario" \
|
|
IOP_HOT_PATH_FAKE_REQUEST_ID="$request_id" \
|
|
IOP_HOT_PATH_FAKE_WORKSPACE="$ws" \
|
|
IOP_HOT_PATH_FAKE_RECORD="$recorded_file" \
|
|
IOP_HOT_PATH_FAKE_INVOCATION_MARKER="$INVOCATION_MARKER" \
|
|
IOP_HOT_PATH_FAKE_OBSERVATION_FILE="$OBSERVATION_FILE" \
|
|
exec "$agent_bin" "${argv_arr[@]}"
|
|
) >"$out_file" 2>"$err_file" &
|
|
child_pid=$!
|
|
if [ "$scenario" = timeout-cancel ]; then
|
|
sleep "$CANCEL_TIMEOUT_SEC"
|
|
if kill -0 "$child_pid" 2>/dev/null; then
|
|
kill -TERM "$child_pid" 2>/dev/null || true
|
|
triggered=true
|
|
target=child_only
|
|
fi
|
|
fi
|
|
wait "$child_pid" 2>/dev/null || child_status=$?
|
|
end_ms=$(date +%s%3N 2>/dev/null | tr -d ' ' || echo 0)
|
|
local duration_ms=$(( end_ms - start_ms ))
|
|
[ "$duration_ms" -lt 0 ] && duration_ms=0
|
|
|
|
local sentinel_survived=false
|
|
if kill -0 "$sentinel_pid" 2>/dev/null; then sentinel_survived=true; fi
|
|
kill "$sentinel_pid" 2>/dev/null || true
|
|
wait "$sentinel_pid" 2>/dev/null || true
|
|
|
|
local snapshot_after
|
|
snapshot_after=$(workspace_snapshot "$ws")
|
|
if [ "$scenario" = write-unavailable ]; then chmod u+w "$ws" 2>/dev/null || true; fi
|
|
if [ "${REQUIRE_RECORDED_ARGV:-false}" = true ]; then
|
|
cmp -s "$argv_file" "$recorded_file" || return 1
|
|
fi
|
|
|
|
local visible_events observation derived outcome terminal cleanup rest
|
|
visible_events=$(parse_visible_events "$agent" "$out_file" "$child_status" "$triggered" "$target") || return 1
|
|
observation=$(capture_appended_observation "$case_id" "$scenario" \
|
|
"$obs_offset_before" "$obs_inode_before") || return 1
|
|
derived=$(derive_case_result "$agent" "$scenario" "$child_status" "$triggered" "$target" \
|
|
"$sentinel_survived" "$visible_events" "$observation" "$snapshot_before" "$snapshot_after") \
|
|
|| return 1
|
|
outcome="${derived%%:*}"; rest="${derived#*:}"
|
|
terminal="${rest%%:*}"; cleanup="${rest##*:}"
|
|
|
|
jq -n \
|
|
--arg id "$case_id" --arg agent "$agent" --arg scenario "$scenario" \
|
|
--arg argv_hash "$argv_hash" --arg outcome "$outcome" --arg terminal "$terminal" \
|
|
--arg cleanup "$cleanup" --argjson process_exit "$child_status" \
|
|
--argjson visible_events "$visible_events" --argjson observation "$observation" \
|
|
--argjson ws_before "$snapshot_before" --argjson ws_after "$snapshot_after" \
|
|
--argjson triggered "$triggered" --arg target "$target" \
|
|
--argjson sentinel_survived "$sentinel_survived" --argjson duration_ms "$duration_ms" '
|
|
{
|
|
id:$id, agent:$agent, scenario:$scenario, argv_hash:$argv_hash,
|
|
process_exit:$process_exit, outcome:$outcome, terminal:$terminal, cleanup:$cleanup,
|
|
visible_events:$visible_events, observation:$observation,
|
|
workspace_before:$ws_before, workspace_after:$ws_after,
|
|
cancellation:{triggered:$triggered,target:$target,sentinel_survived:$sentinel_survived},
|
|
duration_ms:$duration_ms
|
|
}'
|
|
}
|
|
|
|
run_matrix() {
|
|
CASE_RESULTS=()
|
|
for agent in "${AGENTS[@]}"; do
|
|
for scenario in "${SCENARIOS[@]}"; do
|
|
local case_json
|
|
if ! case_json=$(run_case "$agent" "$scenario"); then
|
|
log "case evidence rejected: $agent:$scenario"
|
|
return 1
|
|
fi
|
|
CASE_RESULTS+=("$case_json")
|
|
done
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Manifest assembly, validation, redaction, atomic output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
build_manifest() {
|
|
local cases_array='['
|
|
local first=1
|
|
for c in "${CASE_RESULTS[@]}"; do
|
|
[ "$first" -eq 1 ] || cases_array+=','
|
|
cases_array+="$c"
|
|
first=0
|
|
done
|
|
cases_array+=']'
|
|
|
|
local claude_secret_present=false pi_secret_present=false
|
|
[ -n "${!CLAUDE_SECRET_ENV:-}" ] && claude_secret_present=true
|
|
[ -n "${!PI_SECRET_ENV:-}" ] && pi_secret_present=true
|
|
|
|
local obs_hash ws_root_hash run_id
|
|
# Digest the closed, projected observation evidence actually consumed by the
|
|
# matrix (never the live log file bytes).
|
|
obs_hash=$(printf '%s' "$cases_array" | jq -cS '[.[].observation]' \
|
|
| sha256sum | awk '{printf "sha256:%s", $1}')
|
|
ws_root_hash=$(sha256_str "$(cd "$WORKSPACE_ROOT" && pwd)")
|
|
run_id=$(sha256_str "${SOURCE_HEAD}-${SOURCE_TREE}-${RUNTIME_SHA256}-${cases_array}")
|
|
|
|
local redaction_patterns_json sentinels_seeded_count
|
|
redaction_patterns_json=$(printf '%s\n' "${REDACTION_PATTERN_LABELS[@]}" | jq -R . | jq -sc .)
|
|
sentinels_seeded_count="${SENTINELS_SEEDED:-0}"
|
|
|
|
jq -n \
|
|
--arg schema_version "$SCHEMA_VERSION" \
|
|
--arg run_id "$run_id" \
|
|
--arg head "$SOURCE_HEAD" \
|
|
--arg source_tree "$SOURCE_TREE" \
|
|
--arg script_sha256 "$(sha256_file "$SELF_PATH")" \
|
|
--arg schema_sha256 "$(sha256_file "$SCHEMA_PATH")" \
|
|
--arg runtime_sha256 "$RUNTIME_SHA256" \
|
|
--arg fixture_sha256 "$FIXTURE_SHA256" \
|
|
--arg observation_sha256 "$obs_hash" \
|
|
--arg workspace_root_hash "$ws_root_hash" \
|
|
--arg claude_binary_sha256 "$CLAUDE_BIN_SHA256" \
|
|
--arg pi_binary_sha256 "$PI_BIN_SHA256" \
|
|
--argjson claude_secret_present "$claude_secret_present" \
|
|
--argjson pi_secret_present "$pi_secret_present" \
|
|
--argjson cases "$cases_array" \
|
|
--argjson redaction_patterns "$redaction_patterns_json" \
|
|
--argjson sentinels_seeded "$sentinels_seeded_count" \
|
|
'{
|
|
schema_version: $schema_version,
|
|
run_id: $run_id,
|
|
source: {
|
|
head: $head,
|
|
source_tree: $source_tree,
|
|
script_sha256: $script_sha256,
|
|
schema_sha256: $schema_sha256
|
|
},
|
|
runtime: {
|
|
runtime_sha256: $runtime_sha256,
|
|
fixture_sha256: $fixture_sha256,
|
|
observation_sha256: $observation_sha256,
|
|
workspace_root_hash: $workspace_root_hash
|
|
},
|
|
runner: {
|
|
claude_binary_sha256: $claude_binary_sha256,
|
|
pi_binary_sha256: $pi_binary_sha256,
|
|
claude_secret_present: $claude_secret_present,
|
|
pi_secret_present: $pi_secret_present,
|
|
claude_flags: ["--print","--output-format","stream-json","--include-partial-messages","--no-session-persistence","--bare"],
|
|
pi_flags: ["--provider","--model","--mode","json","--print","--no-session"]
|
|
},
|
|
cases: $cases,
|
|
redaction: {
|
|
patterns: $redaction_patterns,
|
|
sentinels_seeded: $sentinels_seeded,
|
|
matches: 0
|
|
}
|
|
}'
|
|
}
|
|
|
|
# Recursive forbidden-key scan over a JSON document. Emits the final key/index
|
|
# of every jq path and flags any forbidden field name anywhere in the document
|
|
# (defense-in-depth alongside the closed additionalProperties:false schema).
|
|
scan_forbidden_keys() {
|
|
local doc="$1"
|
|
local found
|
|
found=$(jq -r 'paths | .[-1] | tostring' 2>/dev/null <<<"$doc" \
|
|
| grep -E "$FORBIDDEN_KEY_REGEX" | head -1 || true)
|
|
if [ -n "$found" ]; then
|
|
printf 'forbidden-key:%s' "$found"
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
redaction_match_count() {
|
|
local doc="$1"
|
|
local total=0 n
|
|
for pat in "${REDACTION_PATTERNS[@]}"; do
|
|
n=$(printf '%s' "$doc" | grep -E -c -- "$pat" 2>/dev/null || true)
|
|
total=$(( total + n ))
|
|
done
|
|
printf '%s' "$total"
|
|
}
|
|
|
|
persisted_artifacts_are_clean() {
|
|
local path pat
|
|
for path in "$@"; do
|
|
[ -e "$path" ] || continue
|
|
for pat in "${REDACTION_PATTERNS[@]}"; do
|
|
if [ -d "$path" ]; then
|
|
grep -R -I -E -q -- "$pat" "$path" 2>/dev/null && return 1
|
|
elif grep -I -E -q -- "$pat" "$path" 2>/dev/null; then
|
|
return 1
|
|
fi
|
|
done
|
|
done
|
|
return 0
|
|
}
|
|
|
|
validate_schema_fixture() {
|
|
local schema="$1"
|
|
jq -e '
|
|
."$schema" == "https://json-schema.org/draft/2020-12/schema"
|
|
and .type == "object" and .additionalProperties == false
|
|
and .properties.cases.type == "array"
|
|
and .properties.cases.items == false
|
|
and (.properties.cases.prefixItems | length) == 10
|
|
and ([.properties.cases.prefixItems[].properties.id.const] | length == 10)
|
|
and ([.properties.cases.prefixItems[].properties.id.const] | unique | length == 10)
|
|
and all(.properties.cases.prefixItems[];
|
|
."$ref" == "#/$defs/case"
|
|
and (.properties.id.const | test("^(claude|pi):(direct|light-pass|repair|write-unavailable|timeout-cancel)$"))
|
|
and .properties.id.const == (.properties.agent.const + ":" + .properties.scenario.const)
|
|
and (.properties.outcome.const | IN("completed","error","cancelled"))
|
|
and (.properties.terminal.const | IN("success","provider_error","cancelled"))
|
|
and (.properties.cleanup.const | IN("removed","orphan","none"))
|
|
and .properties.observation.type == "array"
|
|
and .properties.observation.items == false
|
|
and (.properties.observation.prefixItems | length > 0)
|
|
and all(.properties.observation.prefixItems[];
|
|
(.properties.stage.const | IN("selector","local","review","cleanup"))
|
|
and (.properties.outcome.const | IN("observed","failed"))
|
|
)
|
|
)
|
|
and (."$defs".case.required | index("process_exit") != null)
|
|
and ."$defs".case.additionalProperties == false
|
|
' "$schema" >/dev/null 2>&1
|
|
}
|
|
|
|
validate_manifest() {
|
|
local schema="$1" doc="$2"
|
|
validate_schema_fixture "$schema" || return 1
|
|
jq -e --slurpfile schema "$schema" '
|
|
def digest: type == "string" and test("^sha256:[0-9a-f]{64}$");
|
|
def exact_keys($v): (keys | sort) == ($v | sort);
|
|
($schema[0].properties.cases.prefixItems | length) as $case_count
|
|
| exact_keys(["cases","redaction","run_id","runner","runtime","schema_version","source"])
|
|
and .schema_version == "1" and (.run_id | digest)
|
|
and (.source | exact_keys(["head","schema_sha256","script_sha256","source_tree"]))
|
|
and (.source.head | test("^[0-9a-f]{7,64}$"))
|
|
and (.source.source_tree | test("^[0-9a-f]{40,64}$"))
|
|
and (.source.script_sha256 | digest) and (.source.schema_sha256 | digest)
|
|
and (.runtime | exact_keys(["fixture_sha256","observation_sha256","runtime_sha256","workspace_root_hash"]))
|
|
and (.runtime.runtime_sha256 | digest) and (.runtime.fixture_sha256 | digest)
|
|
and (.runtime.observation_sha256 | digest) and (.runtime.workspace_root_hash | digest)
|
|
and (.runner | exact_keys(["claude_binary_sha256","claude_flags","claude_secret_present","pi_binary_sha256","pi_flags","pi_secret_present"]))
|
|
and (.runner.claude_binary_sha256 | digest) and (.runner.pi_binary_sha256 | digest)
|
|
and (.runner.claude_secret_present | type == "boolean")
|
|
and (.runner.pi_secret_present | type == "boolean")
|
|
and (.runner.claude_flags == ["--print","--output-format","stream-json","--include-partial-messages","--no-session-persistence","--bare"])
|
|
and (.runner.pi_flags == ["--provider","--model","--mode","json","--print","--no-session"])
|
|
and (.cases | type == "array" and length == $case_count)
|
|
and (.redaction | exact_keys(["matches","patterns","sentinels_seeded"]))
|
|
and .redaction.patterns == ["anthropic_key","pi_key","bearer_value","raw_stdout","raw_prompt"]
|
|
and (.redaction.sentinels_seeded | type == "number")
|
|
and .redaction.sentinels_seeded >= 0
|
|
and .redaction.matches == 0
|
|
' >/dev/null 2>&1 <<<"$doc" || return 1
|
|
|
|
local i case_json schema_row derived recorded expected_observation
|
|
for i in $(seq 0 9); do
|
|
case_json=$(jq -c --argjson i "$i" '.cases[$i]' <<<"$doc") || return 1
|
|
schema_row=$(jq -c --argjson i "$i" '.properties.cases.prefixItems[$i]' "$schema") || return 1
|
|
jq -e --argjson row "$schema_row" '
|
|
((keys | sort) == ["agent","argv_hash","cancellation","cleanup","duration_ms","id","observation","outcome","process_exit","scenario","terminal","visible_events","workspace_after","workspace_before"])
|
|
and .id == $row.properties.id.const
|
|
and .agent == $row.properties.agent.const
|
|
and .scenario == $row.properties.scenario.const
|
|
and .outcome == $row.properties.outcome.const
|
|
and .terminal == $row.properties.terminal.const
|
|
and .cleanup == $row.properties.cleanup.const
|
|
and (.argv_hash | test("^sha256:[0-9a-f]{64}$"))
|
|
and (.process_exit | type == "number") and (.process_exit | floor) == .process_exit
|
|
and .process_exit >= 0 and .process_exit <= 255
|
|
and (.duration_ms | type == "number") and (.duration_ms | floor) == .duration_ms
|
|
and .duration_ms >= 0
|
|
and (.visible_events | type == "array" and length > 0)
|
|
and ([range(0; .visible_events | length)] == [.visible_events[].index])
|
|
and all(.visible_events[];
|
|
((keys | sort) == ["detail","index","kind"])
|
|
and (.kind | IN("system_init","assistant_text","tool_use","tool_result","partial","terminal_success","terminal_error","terminal_cancelled"))
|
|
and (.detail | IN("init","text","workspace_write","review_write","repair_write","workspace_cleanup","tool_call","ok","error","event","delta","success","provider_error","cancelled"))
|
|
)
|
|
and all(.observation[];
|
|
((keys | sort) == ["outcome","request_id","stage"])
|
|
and (.request_id | test("^rid-[0-9a-f]{8,32}$"))
|
|
)
|
|
and ((.workspace_before | keys | sort) == ["artifacts_present","tree_sha256","writable"])
|
|
and ((.workspace_after | keys | sort) == ["artifacts_present","tree_sha256","writable"])
|
|
and (.workspace_before.tree_sha256 | test("^sha256:[0-9a-f]{64}$"))
|
|
and (.workspace_after.tree_sha256 | test("^sha256:[0-9a-f]{64}$"))
|
|
and (.workspace_before.artifacts_present | type == "boolean")
|
|
and (.workspace_after.artifacts_present | type == "boolean")
|
|
and (.workspace_before.writable | type == "boolean")
|
|
and (.workspace_after.writable | type == "boolean")
|
|
and ((.cancellation | keys | sort) == ["sentinel_survived","target","triggered"])
|
|
and .cancellation.triggered == $row.properties.cancellation.properties.triggered.const
|
|
and .cancellation.target == $row.properties.cancellation.properties.target.const
|
|
and (.cancellation.sentinel_survived | type == "boolean")
|
|
' >/dev/null 2>&1 <<<"$case_json" || return 1
|
|
|
|
expected_observation=$(jq -c '[.properties.observation.prefixItems[] | {
|
|
stage:.properties.stage.const,
|
|
outcome:.properties.outcome.const
|
|
}]' <<<"$schema_row") || return 1
|
|
jq -e --argjson expected "$expected_observation" '
|
|
([.observation[] | {stage,outcome}] == $expected)
|
|
' >/dev/null <<<"$case_json" || return 1
|
|
# Each case's observation records must share exactly one runtime request
|
|
# lifecycle in the closed rid- form (correlation is derived from the log,
|
|
# not from a predetermined per-case hash).
|
|
jq -e '
|
|
([.observation[].request_id] | unique | length) == 1
|
|
and all(.observation[]; .request_id | test("^rid-[0-9a-f]{8,32}$"))
|
|
' >/dev/null <<<"$case_json" || return 1
|
|
derived=$(derive_case_result \
|
|
"$(jq -r '.agent' <<<"$case_json")" \
|
|
"$(jq -r '.scenario' <<<"$case_json")" \
|
|
"$(jq -r '.process_exit' <<<"$case_json")" \
|
|
"$(jq -r '.cancellation.triggered' <<<"$case_json")" \
|
|
"$(jq -r '.cancellation.target' <<<"$case_json")" \
|
|
"$(jq -r '.cancellation.sentinel_survived' <<<"$case_json")" \
|
|
"$(jq -c '.visible_events' <<<"$case_json")" \
|
|
"$(jq -c '.observation' <<<"$case_json")" \
|
|
"$(jq -c '.workspace_before' <<<"$case_json")" \
|
|
"$(jq -c '.workspace_after' <<<"$case_json")") || return 1
|
|
recorded=$(jq -r '[.outcome,.terminal,.cleanup] | join(":")' <<<"$case_json")
|
|
[ "$derived" = "$recorded" ] || return 1
|
|
done
|
|
|
|
scan_forbidden_keys "$doc" >/dev/null 2>&1 && return 1
|
|
[ "$(redaction_match_count "$doc")" -eq 0 ] || return 1
|
|
return 0
|
|
}
|
|
|
|
atomic_write() {
|
|
local dest="$1" content="$2"
|
|
local dir
|
|
dir=$(dirname "$dest")
|
|
[ -d "$dir" ] || die_validation "output directory absent: $dir"
|
|
local tmp="$dest.tmp.$$"
|
|
printf '%s\n' "$content" > "$tmp"
|
|
mv -f "$tmp" "$dest"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Top-level modes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_run() {
|
|
validate_inputs_presence
|
|
validate_worktree_fingerprint
|
|
validate_edge_binary_config_fixture_identity
|
|
validate_runner_and_profile_identity
|
|
validate_schema_fixture "$FIXTURE_PATH" \
|
|
|| die_validation "fixture does not implement the closed fixed-matrix schema subset"
|
|
validate_observation_log_preflight
|
|
: > "$INVOCATION_MARKER" 2>/dev/null || true
|
|
RAW_CAPTURE_DIR=$(mktemp -d "$WORKSPACE_ROOT/.e2e-hot-path-capture.XXXXXX") \
|
|
|| die_validation "cannot create disposable raw capture"
|
|
if ! run_matrix; then
|
|
rm -rf "$RAW_CAPTURE_DIR"
|
|
RAW_CAPTURE_DIR=""
|
|
die_validation "execution, terminal, cancellation, observation, or workspace evidence contradicted the fixed scenario"
|
|
fi
|
|
rm -rf "$RAW_CAPTURE_DIR"
|
|
RAW_CAPTURE_DIR=""
|
|
local manifest
|
|
manifest=$(build_manifest)
|
|
validate_manifest "$FIXTURE_PATH" "$manifest" \
|
|
|| die_validation "produced manifest failed supplied schema or runtime correlation validation"
|
|
persisted_artifacts_are_clean "$WORKSPACE_ROOT" \
|
|
|| die_validation "surviving workspace artifact contains raw prompt, output, or credential material"
|
|
atomic_write "$OUTPUT_PATH" "$manifest"
|
|
log "wrote redacted manifest: $OUTPUT_PATH"
|
|
}
|
|
|
|
do_preflight() {
|
|
validate_inputs_presence
|
|
validate_worktree_fingerprint
|
|
validate_edge_binary_config_fixture_identity
|
|
validate_runner_and_profile_identity
|
|
validate_schema_fixture "$FIXTURE_PATH" \
|
|
|| die_validation "fixture does not implement the closed fixed-matrix schema subset"
|
|
validate_observation_log_preflight
|
|
log "preflight ok"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Self-test: credential-free behavioral oracle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Pick a writable parent directory whose filesystem permits execve (the default
|
|
# /tmp is noexec on some sandbox hosts, which would make the fake agent binaries
|
|
# unrunnable). Respects a caller-supplied TMPDIR first, then falls back to the
|
|
# repo parent, repo root, HOME, and /var/tmp, probing each with a tiny script.
|
|
exec_tmp_parent() {
|
|
local candidate probe
|
|
for candidate in "${TMPDIR:-/tmp}" "$(dirname "$REPO_ROOT")" "$REPO_ROOT" "${HOME:-}" "/var/tmp"; do
|
|
[ -n "$candidate" ] || continue
|
|
[ -d "$candidate" ] || continue
|
|
[ -w "$candidate" ] || continue
|
|
probe=$(mktemp -d "$candidate/.e2e-hot-path-probe.XXXXXX" 2>/dev/null) || continue
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$probe/probe"
|
|
chmod 700 "$probe/probe"
|
|
if "$probe/probe" >/dev/null 2>&1; then
|
|
rm -rf "$probe"
|
|
printf '%s' "$candidate"
|
|
return 0
|
|
fi
|
|
rm -rf "$probe"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
write_fake_binary() {
|
|
local path="$1" agent="$2"
|
|
cat > "$path" <<FAKE_EOF
|
|
#!/usr/bin/env bash
|
|
# Deterministic fake ${agent}. Records safe argv, appends production-shaped
|
|
# hot_path_observation records to the live observation log, and emits
|
|
# native-shaped stdout events.
|
|
set -euo pipefail
|
|
marker="\${IOP_HOT_PATH_FAKE_INVOCATION_MARKER:-}"
|
|
if [ -n "\$marker" ]; then
|
|
printf '%s\0' "\$@" >> "\$marker" 2>/dev/null || true
|
|
fi
|
|
record="\${IOP_HOT_PATH_FAKE_RECORD:-}"
|
|
if [ -n "\$record" ]; then
|
|
printf '%s\0' "\$@" >> "\$record" 2>/dev/null || true
|
|
fi
|
|
agent="\${IOP_HOT_PATH_FAKE_AGENT:-${agent}}"
|
|
scenario="\${IOP_HOT_PATH_FAKE_SCENARIO:-direct}"
|
|
rid="\${IOP_HOT_PATH_FAKE_REQUEST_ID:-rid-00000000}"
|
|
ws="\${IOP_HOT_PATH_FAKE_WORKSPACE:-\$PWD}"
|
|
contradiction="\${IOP_HOT_PATH_FAKE_CONTRADICTION:-none}"
|
|
obs_file="\${IOP_HOT_PATH_FAKE_OBSERVATION_FILE:-}"
|
|
obs_mode="\${IOP_HOT_PATH_FAKE_OBS_MODE:-normal}"
|
|
|
|
obs_write() { # event_class stage attempt disposition reason cleanup orphan request_id [msg]
|
|
[ -n "\$obs_file" ] || return 0
|
|
printf '{"msg":"%s","hot_path_event_class":"%s","hot_path_stage_kind":"%s","hot_path_attempt_bucket":"%s","hot_path_disposition":"%s","hot_path_reason":"%s","hot_path_cleanup_outcome":"%s","hot_path_orphan_outcome":"%s","hot_path_request_id":"%s"}\n' \
|
|
"\${9:-hot_path_observation}" "\$1" "\$2" "\$3" "\$4" "\$5" "\$6" "\$7" "\$8" >> "\$obs_file" 2>/dev/null || true
|
|
}
|
|
obs_lifecycle() { # \$1=request_id
|
|
local r="\$1"
|
|
case "\$scenario" in
|
|
direct)
|
|
obs_write dispatch "" "" "" "" "" "" "\$r"
|
|
obs_write terminal "" "" success "" "" "" "\$r" ;;
|
|
light-pass)
|
|
obs_write dispatch "" "" "" "" "" "" "\$r"
|
|
obs_write stage local first tool_turn "" "" "" "\$r"
|
|
obs_write stage local retry success "" "" "" "\$r"
|
|
obs_write light review first "" "" "" "" "\$r"
|
|
obs_write stage review first tool_turn "" "" "" "\$r"
|
|
obs_write stage review retry tool_turn "" "" "" "\$r"
|
|
obs_write stage review retry success "" "" "" "\$r"
|
|
obs_write light cleanup first "" "" "" "" "\$r"
|
|
obs_write cleanup "" "" "" "" success "" "\$r"
|
|
obs_write terminal "" "" success "" "" "" "\$r" ;;
|
|
repair)
|
|
obs_write dispatch "" "" "" "" "" "" "\$r"
|
|
obs_write stage local first tool_turn "" "" "" "\$r"
|
|
obs_write stage local retry success "" "" "" "\$r"
|
|
obs_write light review first "" "" "" "" "\$r"
|
|
obs_write stage review first tool_turn "" "" "" "\$r"
|
|
obs_write stage review retry tool_turn "" "" "" "\$r"
|
|
obs_write stage review retry tool_turn "" "" "" "\$r"
|
|
obs_write light review retry "" "" "" "" "\$r"
|
|
obs_write stage review retry success "" "" "" "\$r"
|
|
obs_write light cleanup first "" "" "" "" "\$r"
|
|
obs_write cleanup "" "" "" "" success "" "\$r"
|
|
obs_write terminal "" "" success "" "" "" "\$r" ;;
|
|
write-unavailable)
|
|
obs_write dispatch "" "" "" provider_error "" "" "\$r" ;;
|
|
timeout-cancel)
|
|
obs_write dispatch "" "" "" "" "" "" "\$r" ;;
|
|
esac
|
|
}
|
|
obs_cancel_lifecycle() {
|
|
[ "\$scenario" = timeout-cancel ] || return 0
|
|
obs_write stage local first caller_cancel "" "" "" "\$rid"
|
|
}
|
|
# Emit the observation lifecycle BEFORE the stdout events so the timeout-cancel
|
|
# scenario has already appended its records before it blocks and is signalled.
|
|
case "\$obs_mode" in
|
|
none) : ;;
|
|
rotate)
|
|
if [ -n "\$obs_file" ]; then
|
|
mv "\$obs_file" "\$obs_file.rot" 2>/dev/null || true
|
|
: > "\$obs_file" 2>/dev/null || true
|
|
fi
|
|
obs_lifecycle "\$rid" ;;
|
|
extra-request)
|
|
obs_lifecycle "\$rid"; obs_write dispatch "" "" "" "" "" "" "rid-otherlifecycle" ;;
|
|
wrong-stage)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid"
|
|
obs_write stage local first success "" "" "" "\$rid"
|
|
obs_write terminal "" "" success "" "" "" "\$rid" ;;
|
|
foreign-message)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid" "not_hot_path_observation" ;;
|
|
unknown-event)
|
|
obs_write unknown "" "" "" "" "" "" "\$rid" ;;
|
|
missing-terminal)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid" ;;
|
|
duplicate-terminal)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid"
|
|
obs_write terminal "" "" success "" "" "" "\$rid"
|
|
obs_write terminal "" "" provider_error "" "" "" "\$rid" ;;
|
|
late-terminal)
|
|
obs_lifecycle "\$rid"
|
|
( sleep 0.05; obs_write terminal "" "" provider_error "" "" "" "\$rid" ) >/dev/null 2>&1 &
|
|
;;
|
|
cleanup-without-success)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid"
|
|
obs_write cleanup "" "" "" "" primary_error "" "\$rid"
|
|
obs_write terminal "" "" success "" "" "" "\$rid" ;;
|
|
unexpected-orphan)
|
|
obs_write dispatch "" "" "" "" "" "" "\$rid"
|
|
obs_write orphan "" "" "" "" "" ttl_expired "\$rid"
|
|
obs_write terminal "" "" success "" "" "" "\$rid" ;;
|
|
immediate-timeout-orphan)
|
|
obs_lifecycle "\$rid"
|
|
if [ "\$scenario" = timeout-cancel ]; then
|
|
obs_write orphan "" "" "" "" "" ttl_expired "\$rid"
|
|
fi ;;
|
|
normal|*) obs_lifecycle "\$rid" ;;
|
|
esac
|
|
|
|
emit() { printf '%s\n' "\$1"; }
|
|
emit_artifact() {
|
|
mkdir -p "\$ws/.iop/job/\$rid" 2>/dev/null || true
|
|
printf 'plan\n' > "\$ws/.iop/job/\$rid/plan.md" 2>/dev/null || true
|
|
printf 'review\n' > "\$ws/.iop/job/\$rid/review.md" 2>/dev/null || true
|
|
}
|
|
remove_artifact() {
|
|
rm -rf "\$ws/.iop/job" 2>/dev/null || true
|
|
}
|
|
emit_cancelled() {
|
|
if [ "\$agent" = "claude" ]; then
|
|
emit '{"type":"result","subtype":"cancelled"}'
|
|
fi
|
|
}
|
|
trap 'obs_cancel_lifecycle; emit_cancelled; exit 143' TERM
|
|
if [ "\$agent" = "claude" ]; then
|
|
case "\$scenario" in
|
|
direct)
|
|
emit '{"type":"system","subtype":"init"}'
|
|
if [ "\$contradiction" = "no-terminal" ]; then exit 0; fi
|
|
emit '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"RAW-OUTPUT-SENTINEL-claude"}]}}'
|
|
if [ "\$contradiction" = "terminal" ]; then
|
|
emit '{"type":"result","subtype":"error"}'
|
|
exit 1
|
|
fi
|
|
emit '{"type":"result","subtype":"success","result":"RAW-OUTPUT-SENTINEL-claude"}'
|
|
if [ "\$contradiction" = "success-exit" ]; then exit 1; fi
|
|
;;
|
|
light-pass)
|
|
emit '{"type":"system","subtype":"init"}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"text","text":"plan"}]}}'
|
|
emit_artifact
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"text","text":"review"}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"review_write","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
remove_artifact
|
|
if [ "\$contradiction" = "empty-reservation" ]; then mkdir -p "\$ws/.iop/job/\$rid"; fi
|
|
if [ "\$contradiction" != "workspace" ]; then printf 'completed\n' > "\$ws/completed.txt"; fi
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"cleanup_delete","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"result","subtype":"success","result":"done"}'
|
|
;;
|
|
repair)
|
|
emit '{"type":"system","subtype":"init"}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"text","text":"plan"}]}}'
|
|
emit_artifact
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"text","text":"defect"}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"review_write","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"repair_write","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
printf 'repaired\n' > "\$ws/seeded.txt"
|
|
remove_artifact
|
|
if [ "\$contradiction" = "empty-reservation" ]; then mkdir -p "\$ws/.iop/job/\$rid"; fi
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"cleanup_delete","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"result","subtype":"success","result":"repaired"}'
|
|
;;
|
|
write-unavailable)
|
|
emit '{"type":"system","subtype":"init"}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":true}]}}'
|
|
emit '{"type":"result","subtype":"error","error":"write_unavailable"}'
|
|
exit 1
|
|
;;
|
|
timeout-cancel)
|
|
emit '{"type":"system","subtype":"init"}'
|
|
emit_artifact
|
|
emit '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"write_plan","input":{}}]}}'
|
|
emit '{"type":"user","message":{"content":[{"type":"tool_result","is_error":false}]}}'
|
|
emit '{"type":"assistant","message":{"content":[{"type":"text","text":"partial"}]}}'
|
|
if [ "\$contradiction" = "cancel" ]; then
|
|
emit '{"type":"result","subtype":"success"}'
|
|
exit 0
|
|
fi
|
|
while :; do sleep 0.1; done
|
|
;;
|
|
esac
|
|
else
|
|
case "\$scenario" in
|
|
direct)
|
|
emit '{"type":"agent_start"}'
|
|
emit '{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"RAW-OUTPUT-SENTINEL-pi"}],"stopReason":"stop"},"assistantMessageEvent":{"type":"text_delta"}}'
|
|
if [ "\$contradiction" = "no-terminal" ]; then exit 0; fi
|
|
if [ "\$contradiction" = "terminal" ]; then
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[],"stopReason":"error"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[],"stopReason":"error"}]}'
|
|
exit 1
|
|
fi
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}'
|
|
if [ "\$contradiction" = "success-exit" ]; then exit 1; fi
|
|
;;
|
|
light-pass)
|
|
emit '{"type":"agent_start"}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}'
|
|
emit_artifact
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-review","toolName":"review_write","args":{"path":".iop/job/review.md"}}'
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-review","toolName":"review_write","result":{},"isError":false}'
|
|
remove_artifact
|
|
if [ "\$contradiction" != "workspace" ]; then printf 'completed\n' > "\$ws/completed.txt"; fi
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-cleanup","toolName":"cleanup_delete","args":{"path":".iop/job"}}'
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-cleanup","toolName":"cleanup_delete","result":{},"isError":false}'
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}'
|
|
;;
|
|
repair)
|
|
emit '{"type":"agent_start"}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}'
|
|
emit_artifact
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-review","toolName":"review_write","args":{"path":".iop/job/review.md"}}'
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-review","toolName":"review_write","result":{},"isError":false}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-repair","toolName":"repair_write","args":{"path":"seeded.txt"}}'
|
|
printf 'repaired\n' > "\$ws/seeded.txt"
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-repair","toolName":"repair_write","result":{},"isError":false}'
|
|
remove_artifact
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-cleanup","toolName":"cleanup_delete","args":{"path":".iop/job"}}'
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-cleanup","toolName":"cleanup_delete","result":{},"isError":false}'
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"repaired"}],"stopReason":"stop"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"repaired"}],"stopReason":"stop"}]}'
|
|
;;
|
|
write-unavailable)
|
|
emit '{"type":"agent_start"}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}'
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":true}'
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[],"stopReason":"error"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[],"stopReason":"error"}]}'
|
|
exit 0
|
|
;;
|
|
timeout-cancel)
|
|
emit '{"type":"agent_start"}'
|
|
emit '{"type":"tool_execution_start","toolCallId":"tool-plan","toolName":"write_plan","args":{"path":".iop/job/plan.md"}}'
|
|
emit_artifact
|
|
emit '{"type":"tool_execution_end","toolCallId":"tool-plan","toolName":"write_plan","result":{},"isError":false}'
|
|
emit '{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"partial"}],"stopReason":"stop"},"assistantMessageEvent":{"type":"text_delta"}}'
|
|
if [ "\$contradiction" = "cancel" ]; then
|
|
emit '{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}}'
|
|
emit '{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"done"}],"stopReason":"stop"}]}'
|
|
exit 0
|
|
fi
|
|
while :; do sleep 0.1; done
|
|
;;
|
|
esac
|
|
fi
|
|
exit 0
|
|
FAKE_EOF
|
|
chmod +x "$path"
|
|
}
|
|
|
|
self_test_assert() {
|
|
# $1 = label, rest = command; fails the self-test if command exits non-zero.
|
|
# The command runs in a subshell so that an explicit `exit` (e.g. the exit 69
|
|
# from die_validation) terminates only the subshell and can be observed here.
|
|
local label="$1"; shift
|
|
if ! ( "$@" ) >/tmp/e2e-hot-path-selftest-out.$$ 2>&1; then
|
|
cat /tmp/e2e-hot-path-selftest-out.$$ >&2 || true
|
|
rm -f /tmp/e2e-hot-path-selftest-out.$$
|
|
die "self-test assertion failed: $label"
|
|
fi
|
|
rm -f /tmp/e2e-hot-path-selftest-out.$$
|
|
log "assertion PASS: $label"
|
|
}
|
|
|
|
self_test_expect_manifest_rejected() {
|
|
local label="$1" schema="$2" doc="$3"
|
|
if validate_manifest "$schema" "$doc" >/dev/null 2>&1; then
|
|
die "self-test assertion failed: $label was accepted"
|
|
fi
|
|
log "assertion PASS: $label rejected"
|
|
}
|
|
|
|
self_test_expect_derive_rejected() {
|
|
local label="$1"; shift
|
|
if derive_case_result "$@" >/dev/null 2>&1; then
|
|
die "self-test assertion failed: $label was accepted"
|
|
fi
|
|
log "assertion PASS: $label rejected"
|
|
}
|
|
|
|
self_test_expect_run_rejected() {
|
|
local label="$1" rc=0
|
|
rm -f "$OUTPUT_PATH"
|
|
( do_run ) >/tmp/e2e-hot-path-negative.$$ 2>&1 || rc=$?
|
|
rm -f /tmp/e2e-hot-path-negative.$$
|
|
[ "$rc" -eq "$EXIT_VALIDATION" ] \
|
|
|| die "self-test assertion failed: $label should exit 69 (got $rc)"
|
|
[ ! -e "$OUTPUT_PATH" ] \
|
|
|| die "self-test assertion failed: $label wrote a manifest"
|
|
log "assertion PASS: $label rejected before manifest output"
|
|
}
|
|
|
|
self_test_expect_preinvocation_reject() {
|
|
local label="$1" rc=0
|
|
: > "$SELF_TEST_MARKER"
|
|
( do_run ) >/tmp/e2e-hot-path-preinv.$$ 2>&1 || rc=$?
|
|
rm -f /tmp/e2e-hot-path-preinv.$$
|
|
[ "$rc" -eq "$EXIT_VALIDATION" ] \
|
|
|| die "self-test assertion failed: $label should exit 69 (got $rc)"
|
|
[ ! -s "$SELF_TEST_MARKER" ] \
|
|
|| die "self-test assertion failed: $label invoked an agent before identity validation"
|
|
log "assertion PASS: $label rejected before invocation"
|
|
}
|
|
|
|
self_test() {
|
|
require_cmd jq
|
|
require_cmd sha256sum
|
|
require_cmd grep
|
|
require_cmd timeout
|
|
|
|
local tmp_parent root
|
|
tmp_parent=$(exec_tmp_parent) \
|
|
|| die "no writable+executable temp parent found; set TMPDIR to an executable dir"
|
|
root=$(mktemp -d "$tmp_parent/e2e-hot-path-self-test.XXXXXX")
|
|
# Ensure all temporary state is removed on any exit (success or failure).
|
|
SELF_TEST_ROOT="$root"
|
|
trap 'rm -rf "$SELF_TEST_ROOT"' EXIT
|
|
local bin_dir="$root/bin" ws_root="$root/ws"
|
|
local claude_bin="$bin_dir/fake-claude" pi_bin="$bin_dir/fake-pi"
|
|
local edge_bin="$bin_dir/fake-edge" edge_config="$root/edge.yaml"
|
|
local pi_config_dir="$root/pi-config"
|
|
local obs_file="$root/hot-path-observation.log"
|
|
local runtime_ev="$root/runtime-evidence.json"
|
|
local out="$root/manifest.json"
|
|
local marker="$root/invocation.marker"
|
|
mkdir -p "$bin_dir" "$ws_root" "$pi_config_dir"
|
|
|
|
SELF_TEST_MARKER="$marker"
|
|
INVOCATION_MARKER="$marker"
|
|
SENTINELS_SEEDED=4
|
|
REQUIRE_RECORDED_ARGV=true
|
|
|
|
write_fake_binary "$claude_bin" claude
|
|
write_fake_binary "$pi_bin" pi
|
|
# A fake Edge binary/config and Pi config dir stand in for the real runtime
|
|
# identity inputs. They are never executed by the self-test.
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$edge_bin"; chmod +x "$edge_bin"
|
|
printf 'edge:\n hot_path:\n enabled: true\n' > "$edge_config"
|
|
printf 'provider: iop-pi-smoke\nbase_url: fake\n' > "$pi_config_dir/config.yaml"
|
|
: > "$obs_file"
|
|
|
|
# Sentinel secret env values (presence-only; never serialized).
|
|
export IOP_FAKE_CLAUDE_KEY='sk-ant-fake-CLAUDE-SENTINEL-0'
|
|
export IOP_FAKE_PI_KEY='pi-fake-PI-SENTINEL-0'
|
|
|
|
# Non-secret base/profile/alias identity inputs (fake; never contacted).
|
|
local base_url="https://iop-hot-smoke.invalid/v1"
|
|
local provider="iop-pi-smoke"
|
|
local direct_model="iop-preset-direct"
|
|
local pass_model="iop-preset-pass"
|
|
local repair_model="iop-preset-repair"
|
|
local slow_model="iop-preset-slow"
|
|
|
|
# Actual identity digests the harness will recompute and compare.
|
|
local script_sha schema_sha head tree fp
|
|
script_sha=$(sha256_file "$SELF_PATH")
|
|
schema_sha=$(sha256_file "$SCHEMA_PATH")
|
|
head=$(git_head)
|
|
tree=$(git_tree)
|
|
# Compute the worktree fingerprint once at top level and export it so every
|
|
# `( do_run )` / `( do_preflight )` subshell inherits the cache instead of
|
|
# re-traversing the tree.
|
|
WORKTREE_FINGERPRINT_CACHE=$(compute_worktree_fingerprint)
|
|
export WORKTREE_FINGERPRINT_CACHE
|
|
fp="$WORKTREE_FINGERPRINT_CACHE"
|
|
local claude_sha pi_sha edge_sha edge_cfg_sha pi_cfg_sha fixture_sha
|
|
claude_sha=$(sha256_file "$claude_bin")
|
|
pi_sha=$(sha256_file "$pi_bin")
|
|
edge_sha=$(sha256_file "$edge_bin")
|
|
edge_cfg_sha=$(sha256_file "$edge_config")
|
|
pi_cfg_sha=$(tree_sha256 "$pi_config_dir")
|
|
fixture_sha="$schema_sha"
|
|
local base_sha provider_sha direct_sha pass_sha repair_sha slow_sha
|
|
base_sha=$(sha256_str "$base_url")
|
|
provider_sha=$(sha256_str "$provider")
|
|
direct_sha=$(sha256_str "$direct_model")
|
|
pass_sha=$(sha256_str "$pass_model")
|
|
repair_sha=$(sha256_str "$repair_model")
|
|
slow_sha=$(sha256_str "$slow_model")
|
|
|
|
jq -n \
|
|
--arg script_sha256 "$script_sha" --arg schema_sha256 "$schema_sha" \
|
|
--arg head "$head" --arg source_tree "$tree" --arg worktree_fingerprint "$fp" \
|
|
--arg claude_binary_sha256 "$claude_sha" --arg pi_binary_sha256 "$pi_sha" \
|
|
--arg edge_binary_sha256 "$edge_sha" --arg edge_config_sha256 "$edge_cfg_sha" \
|
|
--arg pi_config_sha256 "$pi_cfg_sha" --arg fixture_sha256 "$fixture_sha" \
|
|
--arg base_url_sha256 "$base_sha" --arg pi_provider_sha256 "$provider_sha" \
|
|
--arg direct_model_sha256 "$direct_sha" --arg pass_model_sha256 "$pass_sha" \
|
|
--arg repair_model_sha256 "$repair_sha" --arg slow_model_sha256 "$slow_sha" \
|
|
'{
|
|
script_sha256:$script_sha256, schema_sha256:$schema_sha256,
|
|
head:$head, source_tree:$source_tree, worktree_fingerprint:$worktree_fingerprint,
|
|
claude_binary_sha256:$claude_binary_sha256, pi_binary_sha256:$pi_binary_sha256,
|
|
edge_binary_sha256:$edge_binary_sha256, edge_config_sha256:$edge_config_sha256,
|
|
pi_config_sha256:$pi_config_sha256, fixture_sha256:$fixture_sha256,
|
|
base_url_sha256:$base_url_sha256, pi_provider_sha256:$pi_provider_sha256,
|
|
direct_model_sha256:$direct_model_sha256, pass_model_sha256:$pass_model_sha256,
|
|
repair_model_sha256:$repair_model_sha256, slow_model_sha256:$slow_model_sha256
|
|
}' > "$runtime_ev"
|
|
|
|
local bad_digest="sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
|
local ev_fp_bad="$root/ev-fp-bad.json" ev_claude_bad="$root/ev-claude-bad.json"
|
|
local ev_edge_bad="$root/ev-edge-bad.json" ev_edge_cfg_bad="$root/ev-edge-cfg-bad.json"
|
|
local ev_pi_cfg_bad="$root/ev-pi-cfg-bad.json" ev_base_bad="$root/ev-base-bad.json"
|
|
local ev_model_bad="$root/ev-model-bad.json" ev_fixture_bad="$root/ev-fixture-bad.json"
|
|
jq --arg b "$bad_digest" '.worktree_fingerprint=$b' "$runtime_ev" > "$ev_fp_bad"
|
|
jq --arg b "$bad_digest" '.claude_binary_sha256=$b' "$runtime_ev" > "$ev_claude_bad"
|
|
jq --arg b "$bad_digest" '.edge_binary_sha256=$b' "$runtime_ev" > "$ev_edge_bad"
|
|
jq --arg b "$bad_digest" '.edge_config_sha256=$b' "$runtime_ev" > "$ev_edge_cfg_bad"
|
|
jq --arg b "$bad_digest" '.pi_config_sha256=$b' "$runtime_ev" > "$ev_pi_cfg_bad"
|
|
jq --arg b "$bad_digest" '.base_url_sha256=$b' "$runtime_ev" > "$ev_base_bad"
|
|
jq --arg b "$bad_digest" '.slow_model_sha256=$b' "$runtime_ev" > "$ev_model_bad"
|
|
jq --arg b "$bad_digest" '.fixture_sha256=$b' "$runtime_ev" > "$ev_fixture_bad"
|
|
|
|
local -a good_inputs=(
|
|
--claude "$claude_bin" --pi "$pi_bin"
|
|
--runtime-evidence "$runtime_ev" --fixture "$SCHEMA_PATH"
|
|
--base-url "$base_url"
|
|
--direct-model "$direct_model" --pass-model "$pass_model"
|
|
--repair-model "$repair_model" --slow-model "$slow_model"
|
|
--edge-bin "$edge_bin" --edge-config "$edge_config"
|
|
--pi-config-dir "$pi_config_dir" --pi-provider "$provider"
|
|
--observation-file "$obs_file" --workspace-root "$ws_root"
|
|
--output "$out"
|
|
--claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY
|
|
)
|
|
|
|
# --- Positive run through the shared --run path with fake binaries. ---
|
|
: > "$obs_file"
|
|
parse_run_inputs "${good_inputs[@]}"
|
|
self_test_assert "positive do_run exits 0" do_run
|
|
|
|
local manifest
|
|
manifest=$(cat "$out")
|
|
|
|
# --- Manifest validation (shared validator used by --run). ---
|
|
self_test_assert "produced manifest validates against supplied fixture" \
|
|
validate_manifest "$SCHEMA_PATH" "$manifest"
|
|
self_test_assert "production retry observation traces accepted and reduced" \
|
|
bash -c "jq -e 'all(.cases[] | select(.scenario==\"light-pass\" or .scenario==\"repair\"); [.observation[].stage] == [\"selector\",\"local\",\"review\",\"cleanup\"])' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
self_test_assert "native Pi success and error terminals parsed" \
|
|
bash -c "jq -e '(.cases[] | select(.id==\"pi:direct\") | .terminal==\"success\") and (.cases[] | select(.id==\"pi:write-unavailable\") | .terminal==\"provider_error\")' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
self_test_assert "native Pi JSON error with exit 0 accepted" \
|
|
bash -c "jq -e '.cases[] | select(.id==\"pi:write-unavailable\") | .process_exit==0 and .terminal==\"provider_error\" and .outcome==\"error\"' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
self_test_assert "Pi terminal error with exit 0 derivation accepted" \
|
|
derive_case_result pi write-unavailable 0 false none true \
|
|
'[{"index":0,"kind":"tool_use","detail":"workspace_write"},{"index":1,"kind":"tool_result","detail":"error"},{"index":2,"kind":"terminal_error","detail":"provider_error"}]' \
|
|
'[{"request_id":"rid-deadbeef","stage":"selector","outcome":"failed"}]' \
|
|
'{"artifacts_present":false,"writable":false,"tree_sha256":"sha256:before"}' \
|
|
'{"artifacts_present":false,"writable":false,"tree_sha256":"sha256:before"}'
|
|
self_test_expect_derive_rejected "Pi success terminal with nonzero exit" \
|
|
pi direct 1 false none true \
|
|
'[{"index":0,"kind":"assistant_text","detail":"text"},{"index":1,"kind":"terminal_success","detail":"success"}]' \
|
|
'[{"request_id":"rid-deadbeef","stage":"selector","outcome":"observed"}]' \
|
|
'{"artifacts_present":false,"writable":true,"tree_sha256":"sha256:before"}' \
|
|
'{"artifacts_present":false,"writable":true,"tree_sha256":"sha256:before"}'
|
|
self_test_assert "native Pi signal exit 143 reconciled as cancellation" \
|
|
bash -c "jq -e '.cases[] | select(.id==\"pi:timeout-cancel\") | .process_exit==143 and .terminal==\"cancelled\" and .cancellation.target==\"child_only\"' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
self_test_assert "native Pi scenario tool order is visible" \
|
|
bash -c "jq -e '.cases[] | select(.id==\"pi:repair\") | [.visible_events[] | select(.kind==\"tool_use\") | .detail] == [\"workspace_write\",\"review_write\",\"repair_write\",\"workspace_cleanup\"]' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
|
|
# --- Exactly the ten expected case ids in matrix order. ---
|
|
local ids expected_ids
|
|
ids=$(jq -r '.cases[].id' <<<"$manifest")
|
|
expected_ids=$(printf '%s\n' "${EXPECTED_CASE_IDS[@]}")
|
|
self_test_assert "ten unique case ids" \
|
|
bash -c '[ "$1" = "$2" ]' _ "$ids" "$expected_ids"
|
|
|
|
# Exact argv comparison occurred inside every case before disposable raw
|
|
# capture was deleted. No expected/recorded argv or raw observation fragment
|
|
# may survive.
|
|
self_test_assert "raw argv/stdout/observation capture deleted" \
|
|
bash -c '! find "$1" -name "argv-*" -o -name "out-*.jsonl" -o -name "obs-appended-*" | grep -q .' _ "$root"
|
|
|
|
# --- Observations are projected per case from the appended log region. ---
|
|
self_test_assert "observation request ids projected and single per case" \
|
|
bash -c "jq -e 'all(.cases[]; ([.observation[].request_id]|unique|length)==1 and all(.observation[]; .request_id|test(\"^rid-[0-9a-f]{8,32}\$\")))' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
|
|
# --- Success and expected-failure terminals. ---
|
|
self_test_assert "direct cases terminal=success" \
|
|
bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"direct\")|.terminal' <<<\"\$1\" | sort -u)\" = \"success\" ]" _ "$manifest"
|
|
self_test_assert "write-unavailable terminal=provider_error" \
|
|
bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"write-unavailable\")|.terminal' <<<\"\$1\" | sort -u)\" = \"provider_error\" ]" _ "$manifest"
|
|
self_test_assert "timeout-cancel terminal=cancelled" \
|
|
bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"timeout-cancel\")|.terminal' <<<\"\$1\" | sort -u)\" = \"cancelled\" ]" _ "$manifest"
|
|
self_test_assert "process exit status is captured from wait" \
|
|
bash -c "jq -e 'all(.cases[]|select(.terminal==\"success\"); .process_exit==0) and all(.cases[]|select(.agent==\"claude\" and .terminal==\"provider_error\"); .process_exit!=0) and all(.cases[]|select(.agent==\"pi\" and .terminal==\"provider_error\"); .process_exit==0) and all(.cases[]|select(.terminal==\"cancelled\"); .process_exit!=0)' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
|
|
# --- Cleanup/orphan classification. ---
|
|
self_test_assert "light-pass/repair cleanup=removed" \
|
|
bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"light-pass\" or .scenario==\"repair\")|.cleanup' <<<\"\$1\" | sort -u)\" = \"removed\" ]" _ "$manifest"
|
|
self_test_assert "timeout-cancel cleanup=orphan" \
|
|
bash -c "[ \"\$(jq -r '.cases[]|select(.scenario==\"timeout-cancel\")|.cleanup' <<<\"\$1\" | sort -u)\" = \"orphan\" ]" _ "$manifest"
|
|
|
|
# --- Child-only timeout signaling. ---
|
|
self_test_assert "timeout-cancel child_only target" \
|
|
bash -c "jq -e '.cases[]|select(.scenario==\"timeout-cancel\")|.cancellation.target==\"child_only\" and .cancellation.sentinel_survived==true' <<<\"\$1\" >/dev/null" _ "$manifest"
|
|
|
|
# --- Secret absence / zero-match redaction over the real manifest. ---
|
|
self_test_assert "redaction matches == 0 on manifest" \
|
|
bash -c "[ \"\$(grep -E -c -- 'sk-ant-[A-Za-z0-9_-]+|pi-fake-PI-SENTINEL-[0-9]+|IOP_FAKE_CLAUDE_KEY|IOP_FAKE_PI_KEY|Bearer[ ]?[A-Za-z0-9._-]+' <<<\"\$1\" || true)\" = \"0\" ]" _ "$manifest"
|
|
|
|
# --- Redaction is non-vacuous: a leaked sentinel is detected. ---
|
|
local leak
|
|
leak='{"runner":{"note":"sk-ant-fake-CLAUDE-SENTINEL-0 leaked"}}'
|
|
self_test_assert "redaction detects leaked sentinel" \
|
|
bash -c "[ \"\$(grep -E -c -- 'sk-ant-[A-Za-z0-9_-]+|pi-fake-PI-SENTINEL-[0-9]+' <<<\"\$1\" || true)\" != \"0\" ]" _ "$leak"
|
|
|
|
self_test_assert "all surviving harness artifacts are redacted" \
|
|
persisted_artifacts_are_clean "$ws_root" "$out" "$obs_file"
|
|
|
|
local content_probe="$root/content-probe" content_before content_after
|
|
mkdir -p "$content_probe"
|
|
printf 'before\n' > "$content_probe/same-name.txt"
|
|
content_before=$(tree_sha256 "$content_probe")
|
|
printf 'after\n' > "$content_probe/same-name.txt"
|
|
content_after=$(tree_sha256 "$content_probe")
|
|
self_test_assert "workspace digest changes on content-only edit" \
|
|
bash -c '[ "$1" != "$2" ]' _ "$content_before" "$content_after"
|
|
|
|
# --- Schema rejection: a malformed manifest must fail validation. ---
|
|
local bad_manifest
|
|
bad_manifest=$(jq '.cases |= .[0:9]' <<<"$manifest") # only 9 cases
|
|
self_test_expect_manifest_rejected "9-case manifest" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].prompt = "raw"' <<<"$manifest") # forbidden field
|
|
self_test_expect_manifest_rejected "forbidden-field manifest" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].outcome = "bogus"' <<<"$manifest") # bad enum
|
|
self_test_expect_manifest_rejected "bad-enum manifest" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases += [.cases[0]]' <<<"$manifest") # 11 cases / duplicate id
|
|
self_test_expect_manifest_rejected "11-case duplicate manifest" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[1].id = .cases[0].id' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "distinct-row duplicate id" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].agent = "pi"' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "id-agent mismatch" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].scenario = "repair"' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "id-scenario mismatch" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].terminal = "provider_error"' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "terminal-event contradiction" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[0].cancellation.triggered = true' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "cancellation relation mismatch" "$SCHEMA_PATH" "$bad_manifest"
|
|
bad_manifest=$(jq '.cases[1].observation[0].request_id = "rid-deadbeef00"' <<<"$manifest")
|
|
self_test_expect_manifest_rejected "multi-request observation in one case" "$SCHEMA_PATH" "$bad_manifest"
|
|
|
|
local alternate_fixture="$root/alternate-schema.json" malformed_fixture="$root/malformed-schema.json"
|
|
jq '.properties.cases.prefixItems[0].properties.id.const = "pi:direct"' \
|
|
"$SCHEMA_PATH" > "$alternate_fixture"
|
|
self_test_expect_manifest_rejected "alternate fixture changes acceptance" "$alternate_fixture" "$manifest"
|
|
jq '.properties.cases.prefixItems |= .[0:9]' "$SCHEMA_PATH" > "$malformed_fixture"
|
|
self_test_expect_manifest_rejected "malformed nine-row fixture" "$malformed_fixture" "$manifest"
|
|
|
|
# --- Identity mismatches exit 69 before any agent invocation (R1). ---
|
|
local ev
|
|
for ev in \
|
|
"worktree fingerprint mismatch:$ev_fp_bad" \
|
|
"claude binary identity mismatch:$ev_claude_bad" \
|
|
"edge binary identity mismatch:$ev_edge_bad" \
|
|
"edge config identity mismatch:$ev_edge_cfg_bad" \
|
|
"pi config identity mismatch:$ev_pi_cfg_bad" \
|
|
"base url identity mismatch:$ev_base_bad" \
|
|
"scenario alias identity mismatch:$ev_model_bad" \
|
|
"fixture identity mismatch:$ev_fixture_bad"; do
|
|
local label="${ev%%:*}" ev_file="${ev##*:}"
|
|
: > "$obs_file"
|
|
parse_run_inputs \
|
|
--claude "$claude_bin" --pi "$pi_bin" \
|
|
--runtime-evidence "$ev_file" --fixture "$SCHEMA_PATH" \
|
|
--base-url "$base_url" \
|
|
--direct-model "$direct_model" --pass-model "$pass_model" \
|
|
--repair-model "$repair_model" --slow-model "$slow_model" \
|
|
--edge-bin "$edge_bin" --edge-config "$edge_config" \
|
|
--pi-config-dir "$pi_config_dir" --pi-provider "$provider" \
|
|
--observation-file "$obs_file" --workspace-root "$ws_root" \
|
|
--output "$out" \
|
|
--claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY
|
|
self_test_expect_preinvocation_reject "$label"
|
|
done
|
|
|
|
# --- Observation lifecycle and freshness negative controls (R2). ---
|
|
parse_run_inputs "${good_inputs[@]}"
|
|
local saved_observation_wait="$OBSERVATION_WAIT_MSEC"
|
|
OBSERVATION_WAIT_MSEC=250
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=none
|
|
self_test_expect_run_rejected "post-bound lifecycle timeout"
|
|
|
|
# Stale-only: valid-looking records exist before the case offset but nothing
|
|
# is appended for the current case; the run must reject the stale evidence.
|
|
: > "$obs_file"
|
|
printf '{"msg":"hot_path_observation","hot_path_event_class":"dispatch","hot_path_stage_kind":"","hot_path_reason":"","hot_path_request_id":"%s"}\n' \
|
|
"$(request_id_for claude:direct)" >> "$obs_file"
|
|
self_test_expect_run_rejected "stale-only observation rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=rotate
|
|
self_test_expect_run_rejected "rotated/truncated observation rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=extra-request
|
|
self_test_expect_run_rejected "mixed/duplicate request lifecycle rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=wrong-stage
|
|
self_test_expect_run_rejected "wrong observation stage lifecycle rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=foreign-message
|
|
self_test_expect_run_rejected "foreign-message observation lookalike rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=unknown-event
|
|
self_test_expect_run_rejected "unknown production observation event rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=missing-terminal
|
|
self_test_expect_run_rejected "missing observation terminal rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=duplicate-terminal
|
|
self_test_expect_run_rejected "duplicate conflicting observation terminals rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=late-terminal
|
|
self_test_expect_run_rejected "late contradictory observation terminal rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=cleanup-without-success
|
|
self_test_expect_run_rejected "cleanup without successful lifecycle rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=unexpected-orphan
|
|
self_test_expect_run_rejected "unexpected observation orphan rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_OBS_MODE=immediate-timeout-orphan
|
|
self_test_expect_run_rejected "immediate TTL orphan after caller cancellation rejected"
|
|
unset IOP_HOT_PATH_FAKE_OBS_MODE
|
|
OBSERVATION_WAIT_MSEC="$saved_observation_wait"
|
|
|
|
# --- Execution/terminal/workspace contradictions exit 69 (retained). ---
|
|
: > "$obs_file"
|
|
local false_runtime_ev="$root/runtime-evidence-false.json" false_sha
|
|
false_sha=$(sha256_file /bin/false)
|
|
jq --arg c "$false_sha" --arg p "$false_sha" \
|
|
'.claude_binary_sha256=$c | .pi_binary_sha256=$p' "$runtime_ev" > "$false_runtime_ev"
|
|
parse_run_inputs \
|
|
--claude /bin/false --pi /bin/false \
|
|
--runtime-evidence "$false_runtime_ev" --fixture "$SCHEMA_PATH" \
|
|
--base-url "$base_url" \
|
|
--direct-model "$direct_model" --pass-model "$pass_model" \
|
|
--repair-model "$repair_model" --slow-model "$slow_model" \
|
|
--edge-bin "$edge_bin" --edge-config "$edge_config" \
|
|
--pi-config-dir "$pi_config_dir" --pi-provider "$provider" \
|
|
--observation-file "$obs_file" --workspace-root "$ws_root" \
|
|
--output "$out" \
|
|
--claude-secret-env IOP_FAKE_CLAUDE_KEY --pi-secret-env IOP_FAKE_PI_KEY
|
|
self_test_expect_run_rejected "immediate exit with no native output"
|
|
|
|
parse_run_inputs "${good_inputs[@]}"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=no-terminal
|
|
self_test_expect_run_rejected "missing native terminal"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=terminal
|
|
self_test_expect_run_rejected "terminal and scenario contradiction"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=success-exit
|
|
self_test_expect_run_rejected "success terminal with nonzero exit rejected"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=workspace
|
|
self_test_expect_run_rejected "content-insensitive cleanup contradiction"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=empty-reservation
|
|
self_test_expect_run_rejected "empty reserved request directory rejected"
|
|
: > "$obs_file"
|
|
export IOP_HOT_PATH_FAKE_CONTRADICTION=cancel
|
|
self_test_expect_run_rejected "timeout without triggered child cancellation"
|
|
unset IOP_HOT_PATH_FAKE_CONTRADICTION
|
|
|
|
# Native Pi rejects the old OpenAI-choice lookalike and an agent_end that
|
|
# lacks a terminal-capable assistant message.
|
|
local pi_choices_probe="$root/pi-choices-lookalike.jsonl"
|
|
local pi_bad_end_probe="$root/pi-agent-end-without-assistant.jsonl" pi_probe_events
|
|
printf '%s\n' '{"choices":[{"finish_reason":"stop"}]}' > "$pi_choices_probe"
|
|
pi_probe_events=$(parse_visible_events pi "$pi_choices_probe" 0 false none)
|
|
if jq -e 'any(.[]; .kind | startswith("terminal_"))' <<<"$pi_probe_events" >/dev/null; then
|
|
die "self-test assertion failed: OpenAI choices lookalike produced a Pi terminal"
|
|
fi
|
|
log "assertion PASS: OpenAI choices lookalike rejected for Pi"
|
|
printf '%s\n' '{"type":"agent_start"}' '{"type":"agent_end","messages":[]}' > "$pi_bad_end_probe"
|
|
if parse_visible_events pi "$pi_bad_end_probe" 0 false none >/dev/null 2>&1; then
|
|
die "self-test assertion failed: Pi agent_end without assistant was accepted"
|
|
fi
|
|
log "assertion PASS: Pi agent_end without terminal-capable assistant rejected"
|
|
|
|
# --- Preflight validates without invoking agents. ---
|
|
: > "$obs_file"
|
|
rm -f "$marker"
|
|
parse_run_inputs "${good_inputs[@]}"
|
|
self_test_assert "preflight ok" do_preflight
|
|
if [ -f "$marker" ] && [ -s "$marker" ]; then
|
|
die "self-test assertion failed: preflight invoked an agent"
|
|
fi
|
|
|
|
# --- Removal of all temporary state. ---
|
|
rm -rf "$root"
|
|
if [ -d "$root" ]; then
|
|
die "self-test assertion failed: temporary state was not removed"
|
|
fi
|
|
|
|
log "self-test PASSED: exact argv, fixed 2x5 matrix, schema rejection,"
|
|
log " runtime/profile/alias binding mismatch exit 69 before invocation,"
|
|
log " production retry lifecycle closure and negative observation controls,"
|
|
log " native Pi success/error/cancel plus tool order, empty-reservation"
|
|
log " rejection, secret absence, child-only cancellation, cleanup/orphan"
|
|
log " classification, and full cleanup verified with fake agents/runtime only."
|
|
return 0
|
|
}
|
|
|
|
main() {
|
|
local mode="${1:-}"
|
|
case "$mode" in
|
|
--self-test) self_test ;;
|
|
--preflight-only)
|
|
shift
|
|
parse_run_inputs "$@"
|
|
INVOCATION_MARKER="${IOP_HOT_PATH_INVOCATION_MARKER:-/dev/null}"
|
|
SENTINELS_SEEDED=0
|
|
REQUIRE_RECORDED_ARGV=false
|
|
do_preflight
|
|
;;
|
|
--run)
|
|
shift
|
|
parse_run_inputs "$@"
|
|
INVOCATION_MARKER="${IOP_HOT_PATH_INVOCATION_MARKER:-/dev/null}"
|
|
SENTINELS_SEEDED=0
|
|
REQUIRE_RECORDED_ARGV=false
|
|
do_run
|
|
;;
|
|
-h|--help) usage; exit "$EXIT_OK" ;;
|
|
*) usage; exit "$EXIT_USAGE" ;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|