toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/scripts/e2e-single-request-claude.sh
toki d7a150c7fe feat(agent): 단일 요청 실행 경로를 완성한다 
Claude의 단일 Anthropic 요청 안에서 IOP가 Plan, Work, Review와 workspace 도구 실행을 끝내고 실제 dev smoke로 계약을 검증할 수 있어야 한다.\n\n완료 task evidence와 마일스톤 검토 상태도 같은 변경에 고정한다.
2026-08-08 23:35:13 +09:00

2039 lines
90 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
# Credential-safe, closed S12 evidence harness. Self-test uses temporary fakes only.
set -euo pipefail
umask 077
readonly EXIT_USAGE=64
readonly EXIT_VALIDATION=69
readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
readonly SELF="$SCRIPT_DIR/e2e-single-request-claude.sh"
readonly DEFAULT_SCHEMA="$SCRIPT_DIR/fixtures/single-request-claude-smoke-manifest.schema.json"
readonly EXPECTED_RESULT='IOP single-request Claude smoke verified.'
readonly PROMPT='Create smoke-result.txt containing exactly one line: IOP single-request Claude smoke verified. The file must end with a terminating newline. Verify the exact file bytes before finishing.'
readonly MAX_CAPTURE_BYTES=8388608
readonly MAX_FRESH_OBSERVATION_BYTES=16777216
readonly CHILD_SUPERVISOR_GRACE_SECONDS=2
readonly CHILD_SUPERVISOR_WAIT_TICKS=60
RUN_TMP_ROOT=''
RUN_TMP=''
PUBLISH_TMP=''
PUBLISH_PARENT=''
PUBLISH_PREFIX=''
CHILD_PID=''
CLEANING=0
log() {
printf '[single-request-claude-smoke] %s\n' "$*" >&2
}
fail() {
log "validation failed: $*"
exit "$EXIT_VALIDATION"
}
usage() {
printf '%s\n' 'usage: e2e-single-request-claude.sh --self-test | --validate-manifest PATH [--schema PATH] | --preflight-only|--run --claude PATH --runtime-evidence PATH --base-url URL --model ID --edge-bin PATH --node-bin PATH --edge-config PATH --observation-file PATH --metrics-url URL --workspace PATH --output PATH --secret-env NAME [--schema PATH]' >&2
}
sha_string() {
python3 - "$1" <<'PY'
import hashlib, sys
print("sha256:" + hashlib.sha256(sys.argv[1].encode()).hexdigest())
PY
}
sha_file() {
python3 - "$1" <<'PY'
import hashlib, sys
h = hashlib.sha256()
with open(sys.argv[1], "rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
h.update(block)
print("sha256:" + h.hexdigest())
PY
}
canonical_existing() {
python3 - "$1" <<'PY'
import os, sys
path = os.path.realpath(sys.argv[1])
if not os.path.exists(path):
raise SystemExit(1)
print(path)
PY
}
file_identity() {
python3 - "$1" <<'PY'
import os, stat, sys
st = os.lstat(sys.argv[1])
if not stat.S_ISREG(st.st_mode):
raise SystemExit(1)
print(st.st_dev, st.st_ino, st.st_size)
PY
}
file_size() {
python3 - "$1" <<'PY'
import os, sys
print(os.lstat(sys.argv[1]).st_size)
PY
}
prefix_digest() {
python3 - "$1" "$2" <<'PY'
import hashlib, sys
path, raw_size = sys.argv[1:]
remaining = int(raw_size)
h = hashlib.sha256()
with open(path, "rb") as stream:
while remaining:
block = stream.read(min(1024 * 1024, remaining))
if not block:
raise SystemExit(1)
h.update(block)
remaining -= len(block)
print("sha256:" + h.hexdigest())
PY
}
tree_digest() {
python3 - "$1" <<'PY'
import hashlib, os, stat, sys
root = os.path.realpath(sys.argv[1])
records = []
for current, dirs, files in os.walk(root, topdown=True, followlinks=False):
dirs.sort()
files.sort()
for name in dirs + files:
path = os.path.join(current, name)
rel = os.path.relpath(path, root).replace(os.sep, "/")
st = os.lstat(path)
if stat.S_ISREG(st.st_mode):
body = hashlib.sha256()
with open(path, "rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
body.update(block)
kind, value = "file", body.hexdigest()
elif stat.S_ISDIR(st.st_mode):
kind, value = "dir", ""
elif stat.S_ISLNK(st.st_mode):
kind = "link"
value = hashlib.sha256(os.readlink(path).encode()).hexdigest()
else:
kind, value = "special", str(stat.S_IFMT(st.st_mode))
records.append((rel, kind, value))
h = hashlib.sha256()
for record in sorted(records):
h.update("\0".join(record).encode() + b"\0")
print("sha256:" + h.hexdigest())
PY
}
worktree_digest() {
python3 - "$REPO_ROOT" <<'PY'
import hashlib, os, sys
root = os.path.realpath(sys.argv[1])
inputs = [
"apps/edge/internal/openai",
"apps/edge/internal/service",
"apps/node/internal/bootstrap",
"apps/node/internal/workspace",
"packages/go/config",
"scripts/e2e-single-request-claude.sh",
"scripts/fixtures/single-request-claude-smoke-manifest.schema.json",
"Makefile",
]
files = []
for rel in inputs:
path = os.path.join(root, rel)
if os.path.isdir(path):
for current, dirs, names in os.walk(path):
dirs.sort()
for name in sorted(names):
candidate = os.path.join(current, name)
if os.path.isfile(candidate) and not os.path.islink(candidate):
files.append(candidate)
elif os.path.isfile(path) and not os.path.islink(path):
files.append(path)
h = hashlib.sha256()
for path in sorted(files):
rel = os.path.relpath(path, root).replace(os.sep, "/")
body = hashlib.sha256()
with open(path, "rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
body.update(block)
h.update(rel.encode() + b"\0" + body.hexdigest().encode() + b"\0")
print("sha256:" + h.hexdigest())
PY
}
observed_worktree_digest() {
if [ "${IOP_SMOKE_SELF_TEST-}" = '1' ] && [[ "${IOP_SMOKE_TEST_WORKTREE_DIGEST-}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
printf '%s\n' "$IOP_SMOKE_TEST_WORKTREE_DIGEST"
return
fi
worktree_digest
}
current_branch_digest() {
local branch
branch="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null)" || fail 'source branch unavailable'
sha_string "$branch"
}
runner_os() {
python3 - <<'PY'
import platform
print(platform.system().lower())
PY
}
runner_arch() {
python3 - <<'PY'
import platform
print(platform.machine().lower())
PY
}
parse_args() {
MODE=''
SCHEMA="$DEFAULT_SCHEMA"
MANIFEST=''
CLAUDE_BIN=''
RUNTIME_EVIDENCE=''
BASE_URL=''
MODEL=''
EDGE_BIN=''
NODE_BIN=''
EDGE_CONFIG=''
OBSERVATION_FILE=''
METRICS_URL=''
WORKSPACE=''
OUTPUT=''
SECRET_ENV=''
while (($#)); do
case "$1" in
--self-test|--preflight-only|--run)
[ -z "$MODE" ] || { usage; exit "$EXIT_USAGE"; }
MODE="${1#--}"
;;
--validate-manifest)
[ -z "$MODE" ] || { usage; exit "$EXIT_USAGE"; }
MODE='validate-manifest'
shift
(($#)) || { usage; exit "$EXIT_USAGE"; }
MANIFEST="$1"
;;
--schema|--claude|--runtime-evidence|--base-url|--model|--edge-bin|--node-bin|--edge-config|--observation-file|--metrics-url|--workspace|--output|--secret-env)
local key="$1"
shift
(($#)) || { usage; exit "$EXIT_USAGE"; }
case "$key" in
--schema) SCHEMA="$1" ;;
--claude) CLAUDE_BIN="$1" ;;
--runtime-evidence) RUNTIME_EVIDENCE="$1" ;;
--base-url) BASE_URL="$1" ;;
--model) MODEL="$1" ;;
--edge-bin) EDGE_BIN="$1" ;;
--node-bin) NODE_BIN="$1" ;;
--edge-config) EDGE_CONFIG="$1" ;;
--observation-file) OBSERVATION_FILE="$1" ;;
--metrics-url) METRICS_URL="$1" ;;
--workspace) WORKSPACE="$1" ;;
--output) OUTPUT="$1" ;;
--secret-env) SECRET_ENV="$1" ;;
esac
;;
*)
usage
exit "$EXIT_USAGE"
;;
esac
shift
done
[ -n "$MODE" ] || { usage; exit "$EXIT_USAGE"; }
}
validate_schema_contract() {
python3 - "$1" <<'PY'
import json, sys
try:
schema = json.load(open(sys.argv[1]))
except Exception:
raise SystemExit(1)
def closed(value):
if isinstance(value, dict):
if value.get("type") == "object" and value.get("additionalProperties") is not False:
return False
return all(closed(item) for item in value.values())
if isinstance(value, list):
return all(closed(item) for item in value)
return True
root = ["schema_version", "source", "runtime", "ingress", "stages", "terminal", "workspace", "verification", "redaction"]
runtime = [
"runner_os", "runner_arch", "workspace_os", "workspace_arch",
"workspace_root_digest", "workspace_owner_digest", "claude_digest",
"claude_version_digest", "claude_help_digest", "edge_digest",
"edge_version_digest", "node_digest", "node_version_digest",
"config_digest", "config_check_digest",
"schema_digest", "base_url_digest", "public_model_digest",
"stage_engines", "stage_binding_digest",
]
defs = schema.get("$defs", {})
if not closed(schema):
raise SystemExit(1)
if schema.get("type") != "object" or schema.get("additionalProperties") is not False:
raise SystemExit(1)
if schema.get("required") != root:
raise SystemExit(1)
if defs.get("runtime", {}).get("required") != runtime:
raise SystemExit(1)
if defs.get("workspace", {}).get("required") != ["before_digest", "after_digest", "changed"]:
raise SystemExit(1)
if defs.get("verification", {}).get("required") != ["command_digest", "result_file_digest", "exit_code"]:
raise SystemExit(1)
PY
}
validate_manifest() {
python3 - "$1" "$2" <<'PY'
import hashlib, json, re, sys
try:
manifest = json.load(open(sys.argv[1]))
schema = json.load(open(sys.argv[2]))
except Exception:
raise SystemExit(1)
def closed(value):
if isinstance(value, dict):
if value.get("type") == "object" and value.get("additionalProperties") is not False:
return False
return all(closed(item) for item in value.values())
if isinstance(value, list):
return all(closed(item) for item in value)
return True
def exact(value, keys):
return isinstance(value, dict) and set(value) == set(keys)
digest_pattern = re.compile(r"^sha256:[0-9a-f]{64}$")
head_pattern = re.compile(r"^[0-9a-f]{40}$")
platform_pattern = re.compile(r"^[a-z0-9_+-]{1,32}$")
bad_key = re.compile(r"prompt|output|token|key|auth|credential|secret|password|api_key|apikey|endpoint|bearer|cookie|session", re.I)
bad_raw = re.compile(r"SECRET_SENTINEL|RAW_|https?://|Bearer\s|sk-ant-", re.I)
def digest(value):
return isinstance(value, str) and bool(digest_pattern.fullmatch(value))
def safe(value):
if isinstance(value, dict):
return all((key == "forbidden_key_count" or not bad_key.search(key)) and safe(item) for key, item in value.items())
if isinstance(value, list):
return all(safe(item) for item in value)
return not (isinstance(value, str) and bad_raw.search(value))
root_keys = ["schema_version", "source", "runtime", "ingress", "stages", "terminal", "workspace", "verification", "redaction"]
source_keys = ["head", "branch_digest", "worktree_digest"]
runtime_keys = [
"runner_os", "runner_arch", "workspace_os", "workspace_arch",
"workspace_root_digest", "workspace_owner_digest", "claude_digest",
"claude_version_digest", "claude_help_digest", "edge_digest",
"edge_version_digest", "node_digest", "node_version_digest",
"config_digest", "config_check_digest",
"schema_digest", "base_url_digest", "public_model_digest",
"stage_engines", "stage_binding_digest",
]
if not closed(schema) or not exact(manifest, root_keys) or manifest.get("schema_version") != "1" or not safe(manifest):
raise SystemExit(1)
source = manifest["source"]
runtime = manifest["runtime"]
if not exact(source, source_keys) or not head_pattern.fullmatch(source["head"]):
raise SystemExit(1)
if not all(digest(source[key]) for key in ["branch_digest", "worktree_digest"]):
raise SystemExit(1)
if not exact(runtime, runtime_keys):
raise SystemExit(1)
if not platform_pattern.fullmatch(runtime["runner_os"]) or not platform_pattern.fullmatch(runtime["runner_arch"]):
raise SystemExit(1)
if runtime["workspace_os"] not in {"darwin", "linux"} or runtime["workspace_os"] != runtime["runner_os"] or not platform_pattern.fullmatch(runtime["workspace_arch"]):
raise SystemExit(1)
runtime_digests = [key for key in runtime_keys if key.endswith("_digest")]
if not all(digest(runtime[key]) for key in runtime_digests):
raise SystemExit(1)
engines = runtime["stage_engines"]
if engines != ["gemini", "ornith-fast", "gemini"]:
raise SystemExit(1)
owner_material = "|".join([
runtime["workspace_os"], runtime["workspace_arch"], runtime["workspace_root_digest"],
runtime["config_digest"], runtime["node_digest"], runtime["node_version_digest"],
])
owner_digest = "sha256:" + hashlib.sha256(owner_material.encode()).hexdigest()
binding_material = "|".join([runtime["config_digest"], runtime["config_check_digest"], runtime["base_url_digest"], runtime["public_model_digest"], *engines])
binding_digest = "sha256:" + hashlib.sha256(binding_material.encode()).hexdigest()
if runtime["workspace_owner_digest"] != owner_digest or runtime["stage_binding_digest"] != binding_digest:
raise SystemExit(1)
ingress = manifest["ingress"]
if not exact(ingress, ["delta"]) or isinstance(ingress["delta"], bool) or not isinstance(ingress["delta"], int) or ingress["delta"] != 1:
raise SystemExit(1)
stages = manifest["stages"]
if not isinstance(stages, list) or len(stages) != 3:
raise SystemExit(1)
for item, stage, engine in zip(stages, ["plan", "work", "review"], engines):
if not exact(item, ["stage", "engine_family", "duration_ms", "binding_digest"]):
raise SystemExit(1)
if item["stage"] != stage or item["engine_family"] != engine or item["binding_digest"] != binding_digest:
raise SystemExit(1)
if isinstance(item["duration_ms"], bool) or not isinstance(item["duration_ms"], int) or item["duration_ms"] < 0:
raise SystemExit(1)
terminal = manifest["terminal"]
if not exact(terminal, ["count", "stop_reason", "duration_ms"]) or isinstance(terminal["count"], bool) or not isinstance(terminal["count"], int) or terminal["count"] != 1 or terminal["stop_reason"] != "end_turn":
raise SystemExit(1)
if isinstance(terminal["duration_ms"], bool) or not isinstance(terminal["duration_ms"], int) or terminal["duration_ms"] < 0:
raise SystemExit(1)
workspace = manifest["workspace"]
if not exact(workspace, ["before_digest", "after_digest", "changed"]):
raise SystemExit(1)
if not digest(workspace["before_digest"]) or not digest(workspace["after_digest"]) or workspace["before_digest"] == workspace["after_digest"] or workspace["changed"] is not True:
raise SystemExit(1)
verification = manifest["verification"]
if not exact(verification, ["command_digest", "result_file_digest", "exit_code"]):
raise SystemExit(1)
if not digest(verification["command_digest"]) or not digest(verification["result_file_digest"]):
raise SystemExit(1)
if isinstance(verification["exit_code"], bool) or not isinstance(verification["exit_code"], int) or verification["exit_code"] != 0:
raise SystemExit(1)
redaction = manifest["redaction"]
if not exact(redaction, ["forbidden_match_count", "forbidden_key_count"]):
raise SystemExit(1)
if any(isinstance(redaction[key], bool) or not isinstance(redaction[key], int) or redaction[key] != 0 for key in redaction):
raise SystemExit(1)
PY
}
validate_url() {
python3 - "$1" <<'PY'
import sys, urllib.parse
try:
parsed = urllib.parse.urlsplit(sys.argv[1])
except Exception:
raise SystemExit(1)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise SystemExit(1)
if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment:
raise SystemExit(1)
if any(ord(char) < 32 or ord(char) == 127 for char in sys.argv[1]):
raise SystemExit(1)
PY
}
validate_claude_base() {
python3 - "$1" <<'PY'
import sys, urllib.parse
try:
parsed = urllib.parse.urlsplit(sys.argv[1])
except Exception:
raise SystemExit(1)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise SystemExit(1)
if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment:
raise SystemExit(1)
if parsed.path not in {"", "/"}:
raise SystemExit(1)
if any(ord(char) < 32 or ord(char) == 127 for char in sys.argv[1]):
raise SystemExit(1)
PY
}
validate_model() {
python3 - "$1" <<'PY'
import sys
value = sys.argv[1]
if not value or len(value.encode()) > 256 or any(ord(char) < 32 or ord(char) == 127 for char in value):
raise SystemExit(1)
PY
}
validate_observation_source() {
python3 - "$1" <<'PY'
import json, os, sys
path = sys.argv[1]
allowed_messages = {
"bootstrap artifact server listening",
"connected to control plane",
"edge listening for nodes",
"edge_anthropic_pre_ingress_rejection",
"edge_single_request_observation",
"edge_single_request_terminal_rejection",
"node connection established",
"node ready",
"node registration accepted, awaiting dispatch-ready",
"node unregistered",
"openai-compatible server listening",
}
limit = 1024 * 1024
size = os.path.getsize(path)
with open(path, "rb") as stream:
if size > limit:
stream.seek(size - limit)
stream.readline()
body = stream.read(limit)
for raw in body.splitlines():
try:
item = json.loads(raw)
except Exception:
continue
if not isinstance(item, dict):
continue
level, timestamp, message = item.get("level"), item.get("ts"), item.get("msg")
if (
isinstance(level, str)
and isinstance(timestamp, (int, float))
and not isinstance(timestamp, bool)
and isinstance(message, str)
and message in allowed_messages
):
raise SystemExit(0)
raise SystemExit(1)
PY
}
validate_support_tools() {
local tool
if [ "${IOP_SMOKE_SELF_TEST-}" = '1' ] && [ "${IOP_SMOKE_TEST_FAIL_CHECK-}" = 'support-tool' ]; then
fail 'required support executable unavailable'
fi
for tool in python3 curl git cmp mktemp grep sed rm chmod basename dirname printenv env sleep; do
command -v "$tool" >/dev/null 2>&1 || fail 'required support executable unavailable'
done
}
stop_child_supervisor() {
local pid="$1" tick
kill -TERM "$pid" >/dev/null 2>&1 || true
for ((tick = 0; tick < CHILD_SUPERVISOR_WAIT_TICKS; tick++)); do
kill -0 "$pid" >/dev/null 2>&1 || break
sleep 0.05
done
if kill -0 "$pid" >/dev/null 2>&1; then
kill -KILL "$pid" >/dev/null 2>&1 || true
fi
wait "$pid" >/dev/null 2>&1 || true
}
cleanup_run_artifacts() {
[ "$CLEANING" -eq 0 ] || return 0
CLEANING=1
if [ -n "$CHILD_PID" ]; then
stop_child_supervisor "$CHILD_PID"
CHILD_PID=''
fi
if [ -n "$PUBLISH_TMP" ] && [ -n "$PUBLISH_PARENT" ] && [ -n "$PUBLISH_PREFIX" ]; then
local publish_parent publish_name
publish_parent="$(canonical_existing "$(dirname "$PUBLISH_TMP")" 2>/dev/null || true)"
publish_name="$(basename "$PUBLISH_TMP")"
if [ "$publish_parent" = "$PUBLISH_PARENT" ] && [[ "$publish_name" == "$PUBLISH_PREFIX"* ]]; then
rm -f -- "$PUBLISH_TMP"
fi
PUBLISH_TMP=''
fi
if [ -n "$RUN_TMP" ] && [ -n "$RUN_TMP_ROOT" ]; then
local run_parent run_name
run_parent="$(canonical_existing "$(dirname "$RUN_TMP")" 2>/dev/null || true)"
run_name="$(basename "$RUN_TMP")"
if [ "$run_parent" = "$RUN_TMP_ROOT" ] && [[ "$run_name" == single-request-claude.* ]]; then
rm -rf -- "$RUN_TMP"
fi
RUN_TMP=''
fi
CLEANING=0
}
handle_signal() {
local status="$1"
cleanup_run_artifacts
trap - EXIT HUP INT TERM
exit "$status"
}
create_run_context() {
local requested_root
requested_root="${IOP_SMOKE_TMP_ROOT:-${TMPDIR:-/tmp}}"
RUN_TMP_ROOT="$(canonical_existing "$requested_root" 2>/dev/null)" || fail 'temporary root unavailable'
[ -d "$RUN_TMP_ROOT" ] && [ -w "$RUN_TMP_ROOT" ] || fail 'temporary root unavailable'
RUN_TMP="$(mktemp -d "$RUN_TMP_ROOT/single-request-claude.XXXXXX")" || fail 'temporary run directory unavailable'
chmod 700 "$RUN_TMP"
trap cleanup_run_artifacts EXIT
trap 'handle_signal 129' HUP
trap 'handle_signal 130' INT
trap 'handle_signal 143' TERM
}
finish_run_context() {
cleanup_run_artifacts
trap - EXIT HUP INT TERM
}
prepare_output_target() {
[[ "$OUTPUT" == /* ]] || fail 'output target unsafe'
local parent name
parent="$(canonical_existing "$(dirname "$OUTPUT")" 2>/dev/null)" || fail 'output parent unavailable'
name="$(basename "$OUTPUT")"
[ -d "$parent" ] && [ -w "$parent" ] || fail 'output parent unavailable'
[ "$OUTPUT" = "$parent/$name" ] || fail 'output target unsafe'
[ "$name" != '.' ] && [ "$name" != '..' ] && [ -n "$name" ] || fail 'output target unsafe'
[ ! -e "$OUTPUT" ] && [ ! -L "$OUTPUT" ] || fail 'output target already exists'
PUBLISH_PARENT="$parent"
PUBLISH_PREFIX=".$name.tmp."
PUBLISH_TMP="$(mktemp "$PUBLISH_PARENT/$PUBLISH_PREFIX"'XXXXXX')" || fail 'publication temporary unavailable'
}
capture_command() {
local target="$1"
shift
python3 - "$target" "$@" <<'PY'
import os, signal, subprocess, sys
target, *command = sys.argv[1:]
try:
with open(target, "wb") as output:
process = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
try:
status = process.wait(timeout=8)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
raise SystemExit(1)
if status != 0 or os.path.getsize(target) == 0 or os.path.getsize(target) > 65536:
raise SystemExit(1)
except Exception:
raise SystemExit(1)
PY
}
load_runtime() {
local facts
facts="$(python3 - "$RUNTIME_EVIDENCE" <<'PY'
import hashlib, json, re, sys
try:
data = json.load(open(sys.argv[1]))
source = data["source"]
runtime = data["runtime"]
digest = re.compile(r"^sha256:[0-9a-f]{64}$")
platform = re.compile(r"^[a-z0-9_+-]{1,32}$")
source_keys = {"head", "branch_digest", "worktree_digest"}
runtime_keys = {
"runner_os", "runner_arch", "workspace_os", "workspace_arch",
"workspace_root_digest", "workspace_owner_digest", "claude_digest",
"claude_version_digest", "claude_help_digest", "edge_digest",
"edge_version_digest", "node_digest", "node_version_digest",
"config_digest", "config_check_digest",
"schema_digest", "base_url_digest", "public_model_digest",
"stage_engines", "stage_binding_digest",
}
assert set(data) == {"schema_version", "source", "runtime"} and data["schema_version"] == "1"
assert set(source) == source_keys and set(runtime) == runtime_keys
assert re.fullmatch(r"[0-9a-f]{40}", source["head"])
assert all(digest.fullmatch(source[key]) for key in ["branch_digest", "worktree_digest"])
assert platform.fullmatch(runtime["runner_os"]) and platform.fullmatch(runtime["runner_arch"])
assert runtime["workspace_os"] in {"darwin", "linux"}
assert runtime["workspace_os"] == runtime["runner_os"] and platform.fullmatch(runtime["workspace_arch"])
assert all(digest.fullmatch(runtime[key]) for key in runtime_keys if key.endswith("_digest"))
assert runtime["stage_engines"] == ["gemini", "ornith-fast", "gemini"]
owner_material = "|".join([
runtime["workspace_os"], runtime["workspace_arch"], runtime["workspace_root_digest"],
runtime["config_digest"], runtime["node_digest"], runtime["node_version_digest"],
])
owner = "sha256:" + hashlib.sha256(owner_material.encode()).hexdigest()
binding_material = "|".join([runtime["config_digest"], runtime["config_check_digest"], runtime["base_url_digest"], runtime["public_model_digest"], *runtime["stage_engines"]])
binding = "sha256:" + hashlib.sha256(binding_material.encode()).hexdigest()
assert runtime["workspace_owner_digest"] == owner and runtime["stage_binding_digest"] == binding
values = [
source["head"], source["branch_digest"], source["worktree_digest"],
runtime["runner_os"], runtime["runner_arch"], runtime["workspace_os"], runtime["workspace_arch"],
runtime["workspace_root_digest"], runtime["workspace_owner_digest"],
runtime["claude_digest"], runtime["claude_version_digest"], runtime["claude_help_digest"],
runtime["edge_digest"], runtime["edge_version_digest"], runtime["node_digest"], runtime["node_version_digest"],
runtime["config_digest"], runtime["config_check_digest"],
runtime["schema_digest"], runtime["base_url_digest"], runtime["public_model_digest"],
*runtime["stage_engines"], runtime["stage_binding_digest"],
]
print("\t".join(values))
except Exception:
raise SystemExit(1)
PY
)" 2>/dev/null || fail 'runtime evidence invalid'
IFS=$'\t' read -r RHEAD RBRANCH RTREE RRUNNER_OS RRUNNER_ARCH RWORKSPACE_OS RWORKSPACE_ARCH RWORKSPACE_ROOT RWORKSPACE_OWNER RCLAUDE RCLAUDE_VERSION RCLAUDE_HELP REDGE REDGE_VERSION RNODE RNODE_VERSION RCONFIG RCONFIG_CHECK RSCHEMA RBASE RPUBLIC_MODEL RPLAN_ENGINE RWORK_ENGINE RREVIEW_ENGINE RBIND <<<"$facts"
[ -n "$RBIND" ] || fail 'runtime evidence invalid'
}
validate_runtime_snapshot() {
local phase="$1"
local workspace_root claude_version claude_help edge_version node_version config_check
[ "$RHEAD" = "$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null)" ] || fail 'source head mismatch'
[ "$RBRANCH" = "$(current_branch_digest)" ] || fail 'source branch mismatch'
[ "$RTREE" = "$(observed_worktree_digest)" ] || fail 'source worktree mismatch'
[ "$RRUNNER_OS" = "$(runner_os)" ] || fail 'runner operating system mismatch'
[ "$RRUNNER_ARCH" = "$(runner_arch)" ] || fail 'runner architecture mismatch'
workspace_root="$(canonical_existing "$WORKSPACE" 2>/dev/null)" || fail 'workspace unavailable'
[ "$WORKSPACE" = "$workspace_root" ] || fail 'workspace target unsafe'
[ "$RWORKSPACE_ROOT" = "$(sha_string "$workspace_root")" ] || fail 'workspace identity mismatch'
[ "$RCLAUDE" = "$(sha_file "$CLAUDE_BIN")" ] || fail 'Claude identity mismatch'
[ "$REDGE" = "$(sha_file "$EDGE_BIN")" ] || fail 'Edge identity mismatch'
[ "$RNODE" = "$(sha_file "$NODE_BIN")" ] || fail 'Node identity mismatch'
[ "$RCONFIG" = "$(sha_file "$EDGE_CONFIG")" ] || fail 'config identity mismatch'
[ "$RSCHEMA" = "$(sha_file "$SCHEMA")" ] || fail 'schema identity mismatch'
[ "$RBASE" = "$(sha_string "$BASE_URL")" ] || fail 'base URL identity mismatch'
[ "$RPUBLIC_MODEL" = "$(sha_string "$MODEL")" ] || fail 'public model identity mismatch'
claude_version="$RUN_TMP/$phase-claude-version"
claude_help="$RUN_TMP/$phase-claude-help"
edge_version="$RUN_TMP/$phase-edge-version"
node_version="$RUN_TMP/$phase-node-version"
config_check="$RUN_TMP/$phase-config-check"
capture_command "$claude_version" "$CLAUDE_BIN" --version 2>/dev/null || fail 'Claude version check failed'
capture_command "$claude_help" "$CLAUDE_BIN" --help 2>/dev/null || fail 'Claude help check failed'
local flag
for flag in --print --output-format --verbose --no-session-persistence --bare; do
grep -Fq -- "$flag" "$claude_help" || fail 'Claude required flag unavailable'
done
capture_command "$edge_version" "$EDGE_BIN" version 2>/dev/null || fail 'Edge version check failed'
capture_command "$node_version" "$NODE_BIN" version 2>/dev/null || fail 'Node version check failed'
capture_command "$config_check" "$EDGE_BIN" config check --config "$EDGE_CONFIG" 2>/dev/null || fail 'Edge config check failed'
[ "$RCLAUDE_VERSION" = "$(sha_file "$claude_version")" ] || fail 'Claude version identity mismatch'
[ "$RCLAUDE_HELP" = "$(sha_file "$claude_help")" ] || fail 'Claude help identity mismatch'
[ "$REDGE_VERSION" = "$(sha_file "$edge_version")" ] || fail 'Edge version identity mismatch'
[ "$RNODE_VERSION" = "$(sha_file "$node_version")" ] || fail 'Node version identity mismatch'
[ "$RCONFIG_CHECK" = "$(sha_file "$config_check")" ] || fail 'config check identity mismatch'
}
probe_urls() {
python3 - "$BASE_URL" <<'PY'
import sys, urllib.parse
parsed = urllib.parse.urlsplit(sys.argv[1])
origin = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, "", "", ""))
print(origin + "/healthz")
print(origin + "/v1/messages")
print(origin + "/anthropic/v1/models")
PY
}
probe_health_listener() {
local health_url
health_url="$(probe_urls | sed -n '1p')" || fail 'base URL probe derivation failed'
[ -n "$health_url" ] || fail 'base URL probe derivation failed'
curl -fsS --max-time 5 --max-filesize 8192 "$health_url" >"$RUN_TMP/health-body" 2>"$RUN_TMP/health-error" || fail 'health listener unavailable'
}
probe_messages_listener() {
local code messages_url
messages_url="$(probe_urls | sed -n '2p')" || fail 'base URL probe derivation failed'
[ -n "$messages_url" ] || fail 'base URL probe derivation failed'
code="$(curl -sS --max-time 5 --max-filesize 8192 -o "$RUN_TMP/messages-body" -w '%{http_code}' -X OPTIONS "$messages_url" 2>"$RUN_TMP/messages-error")" || fail 'Messages listener unavailable'
case "$code" in
401|405) ;;
*) fail 'Messages listener unavailable' ;;
esac
}
probe_authenticated_model() {
local catalog_url catalog_body
catalog_url="$(probe_urls | sed -n '3p')" || fail 'authenticated model probe derivation failed'
[ -n "$catalog_url" ] || fail 'authenticated model probe derivation failed'
catalog_body="$RUN_TMP/authenticated-model-body"
IOP_SMOKE_AUTH_SECRET="$SECRET_VALUE" \
python3 - <<'PY' | \
curl --config - -fsS --max-time 5 --max-filesize 8192 \
-o "$catalog_body" "$catalog_url" \
2>"$RUN_TMP/authenticated-model-error" || fail 'authenticated model probe rejected'
import os
secret = os.environ.pop("IOP_SMOKE_AUTH_SECRET")
if not secret or "\n" in secret or "\r" in secret:
raise SystemExit(1)
escaped = secret.replace("\\", "\\\\").replace('"', '\\"')
print('header = "x-api-key: ' + escaped + '"')
print('header = "anthropic-version: 2023-06-01"')
PY
python3 - "$catalog_body" "$MODEL" <<'PY' || fail 'authenticated model probe rejected'
import json, sys
try:
path, model = sys.argv[1:]
assert model
body = open(path, "rb").read(8193)
assert len(body) <= 8192
data = json.loads(body)
assert set(data) == {"data", "has_more", "first_id", "last_id"}
assert data["has_more"] is False and isinstance(data["data"], list)
ids = []
for item in data["data"]:
assert set(item) == {"id", "created_at", "display_name", "type"}
assert all(isinstance(item[key], str) for key in item)
assert item["id"] and item["type"] == "model"
ids.append(item["id"])
assert ids.count(model) == 1
except Exception:
raise SystemExit(1)
PY
}
ingress_value() {
local label="$1"
local target="$RUN_TMP/metrics-$label"
curl -fsS --max-time 5 --max-filesize 1048576 "$METRICS_URL" >"$target" 2>"$RUN_TMP/metrics-$label-error" || fail 'metrics endpoint unavailable'
python3 - "$target" <<'PY'
import decimal, re, sys
values = []
pattern = re.compile(r"^iop_anthropic_single_request_ingress_total\s+([^\s]+)\s*$")
for line in open(sys.argv[1]):
match = pattern.match(line)
if match:
try:
value = decimal.Decimal(match.group(1))
except decimal.InvalidOperation:
raise SystemExit(1)
if value < 0 or not value.is_finite():
raise SystemExit(1)
values.append(value)
if len(values) != 1:
raise SystemExit(1)
print(values[0])
PY
}
metric_delta() {
python3 - "$1" "$2" <<'PY'
import decimal, sys
before, after = map(decimal.Decimal, sys.argv[1:])
delta = after - before
if delta != 1:
raise SystemExit(1)
print(delta)
PY
}
capture_observation_snapshot() {
read -r OBS_DEVICE OBS_INODE OBS_SIZE <<<"$(file_identity "$OBSERVATION_FILE" 2>/dev/null)" || fail 'observation log unavailable'
OBS_PREFIX="$(prefix_digest "$OBSERVATION_FILE" "$OBS_SIZE" 2>/dev/null)" || fail 'observation log unavailable'
}
capture_runtime_identity() {
read -r RUNTIME_DEVICE RUNTIME_INODE RUNTIME_SIZE <<<"$(file_identity "$RUNTIME_EVIDENCE" 2>/dev/null)" || fail 'runtime evidence unavailable'
RUNTIME_DIGEST="$(sha_file "$RUNTIME_EVIDENCE")"
}
validate_runtime_identity_unchanged() {
local device inode size
read -r device inode size <<<"$(file_identity "$RUNTIME_EVIDENCE" 2>/dev/null)" || fail 'runtime evidence changed'
[ "$device" = "$RUNTIME_DEVICE" ] && [ "$inode" = "$RUNTIME_INODE" ] && [ "$size" = "$RUNTIME_SIZE" ] || fail 'runtime evidence changed'
[ "$(sha_file "$RUNTIME_EVIDENCE")" = "$RUNTIME_DIGEST" ] || fail 'runtime evidence changed'
}
preflight() {
local ingress_before_auth ingress_after_auth
validate_support_tools
[ -n "$CLAUDE_BIN" ] && [ -n "$RUNTIME_EVIDENCE" ] && [ -n "$BASE_URL" ] && [ -n "$MODEL" ] || fail 'caller input absent'
[ -n "$EDGE_BIN" ] && [ -n "$NODE_BIN" ] && [ -n "$EDGE_CONFIG" ] && [ -n "$OBSERVATION_FILE" ] && [ -n "$METRICS_URL" ] || fail 'caller input absent'
[ -n "$WORKSPACE" ] && [ -n "$OUTPUT" ] && [ -n "$SECRET_ENV" ] || fail 'caller input absent'
[ -f "$CLAUDE_BIN" ] && [ -x "$CLAUDE_BIN" ] && [ ! -L "$CLAUDE_BIN" ] || fail 'Claude executable unavailable'
[ -f "$EDGE_BIN" ] && [ -x "$EDGE_BIN" ] && [ ! -L "$EDGE_BIN" ] || fail 'Edge executable unavailable'
[ -f "$NODE_BIN" ] && [ -x "$NODE_BIN" ] && [ ! -L "$NODE_BIN" ] || fail 'Node executable unavailable'
[ -f "$RUNTIME_EVIDENCE" ] && [ -r "$RUNTIME_EVIDENCE" ] && [ ! -L "$RUNTIME_EVIDENCE" ] || fail 'runtime evidence unavailable'
[ -f "$EDGE_CONFIG" ] && [ -r "$EDGE_CONFIG" ] && [ ! -L "$EDGE_CONFIG" ] || fail 'Edge config unavailable'
[ -f "$OBSERVATION_FILE" ] && [ -r "$OBSERVATION_FILE" ] && [ ! -L "$OBSERVATION_FILE" ] || fail 'observation log unavailable'
[ -f "$SCHEMA" ] && [ -r "$SCHEMA" ] && [ ! -L "$SCHEMA" ] || fail 'manifest schema unavailable'
[ -d "$WORKSPACE" ] && [ -w "$WORKSPACE" ] && [ ! -L "$WORKSPACE" ] || fail 'workspace not writable'
[ ! -e "$WORKSPACE/smoke-result.txt" ] && [ ! -L "$WORKSPACE/smoke-result.txt" ] || fail 'workspace result already exists'
validate_claude_base "$BASE_URL" 2>/dev/null || fail 'base URL invalid'
validate_url "$METRICS_URL" 2>/dev/null || fail 'metrics URL invalid'
validate_model "$MODEL" 2>/dev/null || fail 'public model invalid'
validate_observation_source "$OBSERVATION_FILE" 2>/dev/null || fail 'observation log incompatible'
[[ "$SECRET_ENV" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || fail 'secret variable name invalid'
SECRET_VALUE="$(printenv "$SECRET_ENV" 2>/dev/null || true)"
[ -n "$SECRET_VALUE" ] || fail 'secret variable absent'
validate_schema_contract "$SCHEMA" 2>/dev/null || fail 'manifest schema invalid'
prepare_output_target
load_runtime
capture_runtime_identity
validate_runtime_snapshot preflight
probe_health_listener
probe_messages_listener
ingress_before_auth="$(ingress_value preflight-before-auth)" || fail 'metrics counter unavailable'
probe_authenticated_model
ingress_after_auth="$(ingress_value preflight-after-auth)" || fail 'metrics counter unavailable'
[ "$ingress_before_auth" = "$ingress_after_auth" ] || fail 'authenticated model probe changed ingress'
capture_observation_snapshot
}
classify_claude_failure() {
python3 - "$1" "$2" <<'PY'
import sys
try:
body = b"".join(open(path, "rb").read() for path in sys.argv[1:]).lower()
except Exception:
print("unknown")
raise SystemExit(0)
rules = [
("cli-validation", "cli-usage", [b"requires --verbose", b"unknown option", b"invalid input format", b"input must be provided"]),
("authentication-rejected", "http-401", [b"status 401", b"api error: 401", b"api error 401", b'"status":401', b'"status": 401']),
("authentication-rejected", "authentication", [b"authentication", b"unauthorized", b"invalid api key", b"invalid x-api-key"]),
("transport-failure", "connection-refused", [b"econnrefused", b"connection refused"]),
("transport-failure", "network-timeout", [b"timed out", b"network timeout"]),
("transport-failure", "dns-failure", [b"enotfound"]),
("transport-failure", "fetch-failed", [b"fetch failed"]),
("transport-failure", "tls-certificate", [b"certificate has expired", b"unable to verify the first certificate", b"self signed certificate", b"certificate verify failed", b"unable to get local issuer certificate"]),
("transport-failure", "connection-error", [b"connection error"]),
("api-rejected", "unsupported-beta", [b"unsupported anthropic-beta"]),
("api-rejected", "unknown-field", [b"json: unknown field"]),
("api-rejected", "invalid-thinking", [b"thinking.display", b"adaptive thinking", b"thinking must be enabled"]),
("api-rejected", "invalid-output-config", [b"output_config.effort", b"output_config.format"]),
("api-rejected", "http-400", [b"status 400", b"api error: 400", b"api error 400", b'"status":400', b'"status": 400', b"bad request"]),
("api-rejected", "http-403", [b"status 403", b"api error: 403", b"api error 403", b'"status":403', b'"status": 403', b"forbidden"]),
("api-rejected", "http-404", [b"status 404", b"api error: 404", b"api error 404", b'"status":404', b'"status": 404']),
("api-rejected", "http-429", [b"status 429", b"api error: 429", b"api error 429", b'"status":429', b'"status": 429', b"rate limit"]),
("api-rejected", "api-error", [b"api error"]),
]
for failure_class, reason, patterns in rules:
if any(pattern in body for pattern in patterns):
print(failure_class + "|" + reason)
break
else:
print("unknown|unclassified")
PY
}
run_claude_child() {
local status classification failure_class failure_reason
CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 \
CLAUDE_CODE_DISABLE_TERMINAL_TITLE=1 \
CLAUDE_CODE_MAX_RETRIES=0 \
ANTHROPIC_BASE_URL="$BASE_URL" \
ANTHROPIC_MODEL="$MODEL" \
ANTHROPIC_API_KEY="$SECRET_VALUE" \
IOP_CLAUDE_SUPERVISOR_BIN="$CLAUDE_BIN" \
IOP_CLAUDE_SUPERVISOR_WORKSPACE="$WORKSPACE" \
IOP_CLAUDE_SUPERVISOR_OUT="$RUN_TMP/claude-out" \
IOP_CLAUDE_SUPERVISOR_ERR="$RUN_TMP/claude-err" \
IOP_CLAUDE_SUPERVISOR_PROMPT="$PROMPT" \
IOP_CLAUDE_SUPERVISOR_GRACE_SECONDS="$CHILD_SUPERVISOR_GRACE_SECONDS" \
python3 -c '
import os
import signal
import subprocess
import sys
import time
grace = float(os.environ["IOP_CLAUDE_SUPERVISOR_GRACE_SECONDS"])
command = [
os.environ["IOP_CLAUDE_SUPERVISOR_BIN"],
"--print", "--output-format", "stream-json", "--verbose", "--no-session-persistence", "--bare",
os.environ["IOP_CLAUDE_SUPERVISOR_PROMPT"],
]
child = None
pending_signal = None
termination_signal = None
settling = False
settled = False
test_early_signal = (
os.environ.get("IOP_SMOKE_SELF_TEST") == "1"
and os.environ.get("IOP_CLAUDE_SUPERVISOR_TEST_EARLY_SIGNAL") == "1"
)
def group_exists():
if child is None:
return False
try:
os.killpg(child.pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def wait_for_group_empty(deadline):
while group_exists() and time.monotonic() < deadline:
time.sleep(0.05)
return not group_exists()
def settle_group():
global settled, settling
if child is None or settled or settling:
return
settling = True
try:
if group_exists():
try:
os.killpg(child.pid, signal.SIGTERM)
except ProcessLookupError:
pass
wait_for_group_empty(time.monotonic() + grace)
if group_exists():
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
wait_for_group_empty(time.monotonic() + grace)
if group_exists():
raise RuntimeError("Claude process group did not terminate")
settled = True
finally:
settling = False
def child_preexec():
import resource
resource.setrlimit(resource.RLIMIT_FSIZE, (16384 * 512, 16384 * 512))
if test_early_signal:
os.kill(os.getppid(), signal.SIGTERM)
def terminate(signum, _frame):
global pending_signal, termination_signal
if child is None:
if pending_signal is None:
pending_signal = signum
return
if settling:
return
if termination_signal is None:
termination_signal = signum
settle_group()
try:
child.wait(timeout=grace)
except subprocess.TimeoutExpired:
raise RuntimeError("Claude child did not terminate")
raise SystemExit(128 + termination_signal)
signal.signal(signal.SIGHUP, terminate)
signal.signal(signal.SIGINT, terminate)
signal.signal(signal.SIGTERM, terminate)
with open(os.environ["IOP_CLAUDE_SUPERVISOR_OUT"], "wb") as stdout, open(os.environ["IOP_CLAUDE_SUPERVISOR_ERR"], "wb") as stderr:
child = subprocess.Popen(
command,
cwd=os.environ["IOP_CLAUDE_SUPERVISOR_WORKSPACE"],
stdout=stdout,
stderr=stderr,
start_new_session=True,
env=os.environ.copy(),
preexec_fn=child_preexec,
)
if pending_signal is not None:
terminate(pending_signal, None)
status = child.wait()
settle_group()
raise SystemExit(status)
' &
CHILD_PID=$!
if wait "$CHILD_PID"; then
status=0
else
status=$?
fi
CHILD_PID=''
[ "$(file_size "$RUN_TMP/claude-out")" -le "$MAX_CAPTURE_BYTES" ] || fail 'Claude stdout exceeded capture bound'
[ "$(file_size "$RUN_TMP/claude-err")" -le "$MAX_CAPTURE_BYTES" ] || fail 'Claude stderr exceeded capture bound'
if [ "$status" -ne 0 ]; then
classification="$(classify_claude_failure "$RUN_TMP/claude-out" "$RUN_TMP/claude-err" 2>/dev/null || true)"
failure_class="${classification%%|*}"
failure_reason="${classification#*|}"
case "$failure_class" in
cli-validation|authentication-rejected|transport-failure|api-rejected|unknown) ;;
*) failure_class=unknown ;;
esac
case "$failure_reason" in
cli-usage|http-401|authentication|connection-refused|network-timeout|dns-failure|fetch-failed|tls-certificate|connection-error|unsupported-beta|unknown-field|invalid-thinking|invalid-output-config|http-400|http-403|http-404|http-429|api-error|unclassified) ;;
*) failure_reason=unclassified ;;
esac
fail "Claude invocation failed (status $status class $failure_class reason $failure_reason)"
fi
}
extract_fresh_observation() {
local target="$1"
local length="$2"
python3 - "$OBSERVATION_FILE" "$OBS_SIZE" "$length" "$target" <<'PY'
import sys
source, raw_offset, raw_length, target = sys.argv[1:]
offset, length = int(raw_offset), int(raw_length)
with open(source, "rb") as stream:
stream.seek(offset)
body = stream.read(length)
if len(body) != length:
raise SystemExit(1)
with open(target, "wb") as output:
output.write(body)
PY
}
build_manifest() {
local fresh="$1"
local target="$2"
local before="$3"
local after="$4"
local result_digest="$5"
local verifier_digest="$6"
local verifier_status="$7"
local delta="$8"
python3 - "$fresh" "$target" "$before" "$after" "$result_digest" "$verifier_digest" "$verifier_status" "$delta" \
"$RHEAD" "$RBRANCH" "$RTREE" "$RRUNNER_OS" "$RRUNNER_ARCH" "$RWORKSPACE_OS" "$RWORKSPACE_ARCH" \
"$RWORKSPACE_ROOT" "$RWORKSPACE_OWNER" "$RCLAUDE" "$RCLAUDE_VERSION" "$RCLAUDE_HELP" "$REDGE" "$REDGE_VERSION" \
"$RNODE" "$RNODE_VERSION" "$RCONFIG" "$RCONFIG_CHECK" "$RSCHEMA" "$RBASE" "$RPUBLIC_MODEL" \
"$RPLAN_ENGINE" "$RWORK_ENGINE" "$RREVIEW_ENGINE" "$RBIND" <<'PY'
import collections, json, re, sys
(
fresh, target, before, after, result_digest, verifier_digest, raw_verifier_status, raw_delta,
head, branch, tree, runner_os, runner_arch, workspace_os, workspace_arch,
workspace_root, workspace_owner, claude, claude_version, claude_help, edge, edge_version,
node, node_version, config, config_check, schema, base, public_model,
plan_engine, work_engine, review_engine, binding,
) = sys.argv[1:]
try:
verifier_status = int(raw_verifier_status)
assert verifier_status == 0 and float(raw_delta) == 1
allowed = {
"level", "ts", "time", "caller", "logger", "msg", "message", "correlation",
"event_class", "stage", "operation", "outcome", "error_class", "duration_ms",
"tool_count", "has_result",
}
observations = []
with open(fresh) as stream:
for line in stream:
try:
item = json.loads(line)
except Exception:
continue
if item.get("msg", item.get("message")) != "edge_single_request_observation":
continue
assert set(item).issubset(allowed)
correlation = item.get("correlation")
assert isinstance(correlation, str) and re.fullmatch(r"sr-[a-z0-9-]{1,64}", correlation)
observations.append(item)
groups = collections.defaultdict(list)
for item in observations:
groups[item["correlation"]].append(item)
candidates = []
for records in groups.values():
request = [item for item in records if item.get("event_class") == "request" and item.get("operation") == "total"]
stages = [item for item in records if item.get("event_class") == "stage"]
terminals = [item for item in records if item.get("event_class") == "terminal"]
if len(request) == 1 and len(stages) == 3 and len(terminals) == 1:
candidates.append((request[0], stages, terminals[0]))
assert len(candidates) == 1
request, stages, terminal = candidates[0]
engines = [plan_engine, work_engine, review_engine]
packed = []
for item, stage, engine in zip(stages, ["plan", "work", "review"], engines):
assert item.get("stage") == stage and item.get("operation") == stage and item.get("outcome") == "success"
assert item.get("error_class") in (None, "", "none")
duration = item.get("duration_ms")
assert isinstance(duration, int) and not isinstance(duration, bool) and duration >= 0
packed.append({"stage": stage, "engine_family": engine, "duration_ms": duration, "binding_digest": binding})
assert request.get("outcome") == "success" and request.get("error_class") in (None, "", "none")
request_duration = request.get("duration_ms")
assert request_duration == 0
assert terminal.get("operation") == "terminal" and terminal.get("outcome") == "success"
assert terminal.get("error_class") in (None, "", "none")
assert terminal.get("has_result") is True
total_duration = terminal.get("duration_ms")
assert isinstance(total_duration, int) and not isinstance(total_duration, bool) and total_duration >= 0
manifest = {
"schema_version": "1",
"source": {"head": head, "branch_digest": branch, "worktree_digest": tree},
"runtime": {
"runner_os": runner_os,
"runner_arch": runner_arch,
"workspace_os": workspace_os,
"workspace_arch": workspace_arch,
"workspace_root_digest": workspace_root,
"workspace_owner_digest": workspace_owner,
"claude_digest": claude,
"claude_version_digest": claude_version,
"claude_help_digest": claude_help,
"edge_digest": edge,
"edge_version_digest": edge_version,
"node_digest": node,
"node_version_digest": node_version,
"config_digest": config,
"config_check_digest": config_check,
"schema_digest": schema,
"base_url_digest": base,
"public_model_digest": public_model,
"stage_engines": engines,
"stage_binding_digest": binding,
},
"ingress": {"delta": 1},
"stages": packed,
"terminal": {"count": 1, "stop_reason": "end_turn", "duration_ms": total_duration},
"workspace": {"before_digest": before, "after_digest": after, "changed": True},
"verification": {"command_digest": verifier_digest, "result_file_digest": result_digest, "exit_code": verifier_status},
"redaction": {"forbidden_match_count": 0, "forbidden_key_count": 0},
}
with open(target, "w") as output:
json.dump(manifest, output, sort_keys=True, separators=(",", ":"))
output.write("\n")
except Exception:
raise SystemExit(1)
PY
}
validate_raw_redaction() {
python3 - "$PUBLISH_TMP" "$SECRET_VALUE" "$BASE_URL" "$MODEL" "$WORKSPACE" "$OUTPUT" "$EXPECTED_RESULT" <<'PY'
import sys
body = open(sys.argv[1], "rb").read()
for value in sys.argv[2:]:
encoded = value.encode()
if len(encoded) >= 4 and encoded in body:
raise SystemExit(1)
PY
}
atomic_publish_no_replace() {
python3 - "$1" "$2" <<'PY'
import ctypes, os, platform, sys
source, target = map(os.fsencode, sys.argv[1:])
with open(sys.argv[1], "rb") as stream:
os.fsync(stream.fileno())
libc = ctypes.CDLL(None, use_errno=True)
system = platform.system().lower()
if system == "linux" and hasattr(libc, "renameat2"):
operation = libc.renameat2
operation.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
operation.restype = ctypes.c_int
status = operation(-100, source, -100, target, 1)
elif system == "darwin" and hasattr(libc, "renamex_np"):
operation = libc.renamex_np
operation.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
operation.restype = ctypes.c_int
status = operation(source, target, 0x00000004)
else:
raise SystemExit(1)
if status != 0:
raise SystemExit(1)
try:
descriptor = os.open(os.path.dirname(sys.argv[2]), os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
except OSError:
pass
PY
}
run_once() {
create_run_context
preflight
local before_metric before_workspace after_metric after_workspace delta
local current_device current_inode current_size fresh_size result_digest verifier_digest verifier_status
before_metric="$(ingress_value before)" || fail 'metrics counter unavailable'
before_workspace="$(tree_digest "$WORKSPACE")"
run_claude_child
validate_runtime_identity_unchanged
validate_runtime_snapshot post
read -r current_device current_inode current_size <<<"$(file_identity "$OBSERVATION_FILE" 2>/dev/null)" || fail 'observation log unavailable'
[ "$current_device" = "$OBS_DEVICE" ] && [ "$current_inode" = "$OBS_INODE" ] && [ "$current_size" -ge "$OBS_SIZE" ] || fail 'observation log rotated or truncated'
[ "$(prefix_digest "$OBSERVATION_FILE" "$OBS_SIZE")" = "$OBS_PREFIX" ] || fail 'observation log rotated or truncated'
fresh_size=$((current_size - OBS_SIZE))
[ "$fresh_size" -le "$MAX_FRESH_OBSERVATION_BYTES" ] || fail 'fresh observation exceeded capture bound'
extract_fresh_observation "$RUN_TMP/fresh-observation" "$fresh_size" || fail 'fresh observation extraction failed'
after_metric="$(ingress_value after)" || fail 'metrics counter unavailable'
delta="$(metric_delta "$before_metric" "$after_metric" 2>/dev/null)" || fail 'ingress delta mismatch'
after_workspace="$(tree_digest "$WORKSPACE")"
[ "$before_workspace" != "$after_workspace" ] || fail 'workspace did not change'
[ -f "$WORKSPACE/smoke-result.txt" ] && [ -r "$WORKSPACE/smoke-result.txt" ] && [ ! -L "$WORKSPACE/smoke-result.txt" ] || fail 'workspace verification file absent'
printf '%s\n' "$EXPECTED_RESULT" >"$RUN_TMP/expected-result"
if cmp -s -- "$RUN_TMP/expected-result" "$WORKSPACE/smoke-result.txt"; then
verifier_status=0
else
verifier_status=$?
fi
if [ "${IOP_SMOKE_SELF_TEST-}" = '1' ] && [ "${IOP_SMOKE_TEST_FORCE_VERIFIER_FAILURE-}" = '1' ]; then
verifier_status=1
fi
[ "$verifier_status" -eq 0 ] || fail 'workspace verification failed'
result_digest="$(sha_file "$WORKSPACE/smoke-result.txt")"
verifier_digest="$(sha_string "cmp-v1|$(sha_file "$RUN_TMP/expected-result")")"
build_manifest "$RUN_TMP/fresh-observation" "$PUBLISH_TMP" "$before_workspace" "$after_workspace" "$result_digest" "$verifier_digest" "$verifier_status" "$delta" || fail 'fresh evidence does not satisfy S12 harness contract'
validate_manifest "$PUBLISH_TMP" "$SCHEMA" || fail 'generated manifest invalid'
validate_raw_redaction || fail 'generated manifest contains forbidden raw evidence'
[ ! -e "$OUTPUT" ] && [ ! -L "$OUTPUT" ] || fail 'output target changed during run'
atomic_publish_no_replace "$PUBLISH_TMP" "$OUTPUT" || fail 'manifest publication failed'
PUBLISH_TMP=''
finish_run_context
log 'run manifest validated and written (redacted evidence only)'
}
preflight_only() {
create_run_context
preflight
finish_run_context
log 'preflight passed without a Claude invocation'
}
self_test() {
mkdir -p "$REPO_ROOT/build"
python3 - "$SELF" "$REPO_ROOT" "$DEFAULT_SCHEMA" "$EXPECTED_RESULT" "$PROMPT" <<'PY'
import copy
import hashlib
import http.server
import json
import os
import pathlib
import platform
import shutil
import signal
import socketserver
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
SELF, REPO_ROOT, SCHEMA, EXPECTED_RESULT, PROMPT = sys.argv[1:]
REPO_ROOT = pathlib.Path(REPO_ROOT).resolve()
SCHEMA = pathlib.Path(SCHEMA).resolve()
class TestFailure(Exception):
pass
def check(condition, message):
if not condition:
raise TestFailure(message)
def sha_text(value):
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
def sha_file(path):
return "sha256:" + hashlib.sha256(pathlib.Path(path).read_bytes()).hexdigest()
def recompute(runtime):
values = runtime["runtime"]
owner_material = "|".join([
values["workspace_os"], values["workspace_arch"], values["workspace_root_digest"],
values["config_digest"], values["node_digest"], values["node_version_digest"],
])
values["workspace_owner_digest"] = sha_text(owner_material)
binding_material = "|".join([values["config_digest"], values["config_check_digest"], values["base_url_digest"], values["public_model_digest"], *values["stage_engines"]])
values["stage_binding_digest"] = sha_text(binding_material)
CLAUDE_VERSION = "Claude fake 1\n"
CLAUDE_HELP = "--print\n--output-format\n--verbose\n--no-session-persistence\n--bare\n"
EDGE_VERSION = "IOP Edge fake 1\n"
NODE_VERSION = "IOP Node fake 1\n"
CONFIG_CHECK = "configuration valid\n"
SELF_TEST_WORKTREE = sha_text("credential-free-self-test-worktree-v1")
CLAUDE_FAKE = r'''#!/usr/bin/env bash
set -euo pipefail
hash_text(){ if command -v sha256sum >/dev/null 2>&1;then printf %s "$1"|sha256sum|awk '{print "sha256:"$1}';else printf %s "$1"|shasum -a 256|awk '{print "sha256:"$1}';fi; }
if [ "${1-}" = '--version' ];then
[ "${IOP_SMOKE_FAKE_CLAUDE_VERSION-}" != 'fail' ] || exit 20
printf 'Claude fake 1\n'
exit 0
fi
if [ "${1-}" = '--help' ];then
[ "${IOP_SMOKE_FAKE_CLAUDE_HELP-}" != 'fail' ] || exit 21
if [ "${IOP_SMOKE_FAKE_CLAUDE_HELP-}" = 'missing' ];then printf '%s\n' '--print' '--output-format';else printf '%s\n' '--print' '--output-format' '--verbose' '--no-session-persistence' '--bare';fi
exit 0
fi
verbose_count=0
last_arg=''
for arg in "$@";do
last_arg="$arg"
if [ "$arg" = '--verbose' ];then verbose_count=$((verbose_count+1));fi
done
[ "$verbose_count" -eq 1 ] || exit 26
[ "$(hash_text "$last_arg")" = "$IOP_SMOKE_FAKE_EXPECT_PROMPT_DIGEST" ] || exit 32
printf '1\n' >>"$IOP_SMOKE_FAKE_MARKER"
[ "$(hash_text "${ANTHROPIC_MODEL-}")" = "$IOP_SMOKE_FAKE_EXPECT_MODEL_DIGEST" ] || exit 22
[ "$(hash_text "${ANTHROPIC_BASE_URL-}")" = "$IOP_SMOKE_FAKE_EXPECT_BASE_DIGEST" ] || exit 23
[ -n "${ANTHROPIC_API_KEY-}" ] || exit 24
[ "${CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS-}" = '1' ] || exit 27
[ "${CLAUDE_CODE_DISABLE_TERMINAL_TITLE-}" = '1' ] || exit 29
[ "${CLAUDE_CODE_MAX_RETRIES-}" = '0' ] || exit 28
behavior="${IOP_SMOKE_FAKE_BEHAVIOR-success}"
case "$behavior" in
claude-failure-cli) printf '%s\n' 'Error: stream-json requires --verbose' >&2; exit 25 ;;
claude-failure-auth) printf '%s\n' 'Error: authentication rejected' >&2; exit 25 ;;
claude-failure-transport) printf '%s\n' 'Error: connection refused' >&2; exit 25 ;;
claude-failure-connection) printf '%s\n' 'Error: API Error: Connection error. SECRET_CONNECTION_MARKER' >&2; exit 25 ;;
claude-failure-tls) printf '%s\n' 'Error: API Error: Connection error. certificate has expired SECRET_CERT_MARKER' >&2; exit 25 ;;
claude-failure-api) printf '%s\n' 'Error: status 429' >&2; exit 25 ;;
claude-failure-api-400) printf '%s\n' 'Error: API Error: 400 rejected-field-marker' >&2; exit 25 ;;
claude-failure-api-400-beta) printf '%s\n' 'Error: API Error: 400 unsupported anthropic-beta "SECRET_REJECTED_BETA"' >&2; exit 25 ;;
claude-failure-api-400-field) printf '%s\n' 'Error: API Error: 400 decode Messages request: json: unknown field "SECRET_FIELD_MARKER"' >&2; exit 25 ;;
claude-failure-api-400-thinking) printf '%s\n' 'Error: API Error: 400 thinking.display SECRET_THINKING_MARKER' >&2; exit 25 ;;
claude-failure-api-400-output) printf '%s\n' 'Error: API Error: 400 output_config.effort SECRET_OUTPUT_MARKER' >&2; exit 25 ;;
claude-failure-unknown) printf '%s\n' 'opaque failure' >&2; exit 25 ;;
esac
if [ "$behavior" = 'leader-exit' ];then
( trap '' TERM INT HUP; while :; do sleep 1; done ) &
printf '%s\n' "$!" >"$IOP_SMOKE_FAKE_DESCENDANT"
exit 0
fi
if [ "$behavior" = 'term-resistant' ] || [ "$behavior" = 'early-signal' ];then
( trap '' TERM INT HUP; while :; do sleep 1; done ) &
printf '%s\n' "$!" >"$IOP_SMOKE_FAKE_DESCENDANT"
trap '' TERM INT HUP
while :; do sleep 1; done
fi
emit(){
request_duration=0
terminal_duration=11
terminal_result=',"has_result":true'
case "$behavior" in
timing-swapped) request_duration=11; terminal_duration=0 ;;
terminal-no-result) terminal_result='' ;;
terminal-false-result) terminal_result=',"has_result":false' ;;
esac
for line in \
"{\"msg\":\"edge_single_request_observation\",\"correlation\":\"sr-selftest\",\"event_class\":\"request\",\"stage\":\"none\",\"operation\":\"total\",\"outcome\":\"success\",\"error_class\":\"none\",\"duration_ms\":$request_duration}" \
'{"msg":"edge_single_request_observation","correlation":"sr-selftest","event_class":"stage","stage":"plan","operation":"plan","outcome":"success","error_class":"none","duration_ms":3}' \
'{"msg":"edge_single_request_observation","correlation":"sr-selftest","event_class":"stage","stage":"work","operation":"work","outcome":"success","error_class":"none","duration_ms":5}' \
'{"msg":"edge_single_request_observation","correlation":"sr-selftest","event_class":"stage","stage":"review","operation":"review","outcome":"success","error_class":"none","duration_ms":2}' \
"{\"msg\":\"edge_single_request_observation\",\"correlation\":\"sr-selftest\",\"event_class\":\"terminal\",\"stage\":\"none\",\"operation\":\"terminal\",\"outcome\":\"success\",\"error_class\":\"none\",\"duration_ms\":$terminal_duration$terminal_result}"
do printf '%s\n' "$line" >>"$IOP_SMOKE_FAKE_OBSERVATION";done
}
if [ "$behavior" = 'rotated' ];then : >"$IOP_SMOKE_FAKE_OBSERVATION";fi
if [ "$behavior" != 'stale' ];then emit;fi
printf 'iop_anthropic_single_request_ingress_total 1\n' >"$IOP_SMOKE_FAKE_METRICS"
case "$behavior" in
no-change) ;;
wrong-content) printf 'wrong\n' >smoke-result.txt ;;
*) printf 'IOP single-request Claude smoke verified.\n' >smoke-result.txt ;;
esac
if [ "$behavior" = 'config-after' ];then printf 'changed\n' >>"$IOP_SMOKE_FAKE_CONFIG";fi
if [ "$behavior" = 'node-after' ];then printf '# changed\n' >>"$IOP_SMOKE_FAKE_NODE";fi
if [ "$behavior" = 'runtime-after' ];then printf '\n' >>"$IOP_SMOKE_FAKE_RUNTIME";fi
if [ "$behavior" = 'output-race' ];then printf 'concurrent owner\n' >"$IOP_SMOKE_FAKE_OUTPUT";fi
'''
EDGE_FAKE = r'''#!/usr/bin/env bash
set -euo pipefail
if [ "${1-}" = 'version' ];then [ "${IOP_SMOKE_FAKE_EDGE_VERSION-}" != 'fail' ] || exit 30;printf 'IOP Edge fake 1\n';exit 0;fi
if [ "${1-}" = 'config' ] && [ "${2-}" = 'check' ] && [ "${3-}" = '--config' ] && [ -f "${4-}" ];then [ "${IOP_SMOKE_FAKE_CONFIG_CHECK-}" != 'fail' ] || exit 31;printf 'configuration valid\n';exit 0;fi
exit 32
'''
NODE_FAKE = r'''#!/usr/bin/env bash
set -euo pipefail
if [ "${1-}" = 'version' ];then [ "${IOP_SMOKE_FAKE_NODE_VERSION-}" != 'fail' ] || exit 40;printf 'IOP Node fake 1\n';exit 0;fi
exit 41
'''
class Listener(http.server.BaseHTTPRequestHandler):
metrics_path = None
model = "MODEL_SENTINEL"
secret = "SECRET_SENTINEL_VALUE"
mode = "ok"
def do_GET(self):
if self.path == "/healthz":
if Listener.mode == "health-fail":
self.send_response(503)
self.end_headers()
return
self.send_response(200)
self.end_headers()
self.wfile.write(b"ok\n")
return
if self.path == "/metrics":
if Listener.mode == "metrics-fail":
self.send_response(503)
self.end_headers()
return
self.send_response(200)
self.end_headers()
self.wfile.write(pathlib.Path(Listener.metrics_path).read_bytes())
return
if self.path == "/anthropic/v1/models":
if Listener.mode == "catalog-auth-reject" or self.headers.get("x-api-key") != Listener.secret:
self.send_response(401)
self.end_headers()
return
if self.headers.get("anthropic-version") != "2023-06-01":
self.send_response(400)
self.end_headers()
return
if Listener.mode == "catalog-malformed":
body = b"not-json\n"
else:
model = "OTHER_MODEL" if Listener.mode == "catalog-model-missing" else Listener.model
body = json.dumps({
"data": [{"id": model, "created_at": "2024-01-01T00:00:00Z", "display_name": model, "type": "model"}],
"has_more": False, "first_id": model, "last_id": model,
}).encode()
if Listener.mode == "catalog-ingress-change":
pathlib.Path(Listener.metrics_path).write_text("iop_anthropic_single_request_ingress_total 1\n")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(404)
self.end_headers()
def do_OPTIONS(self):
if self.path == "/v1/messages":
status = {
"messages-auth": 401,
"messages-fail": 503,
"messages-missing": 404,
}.get(Listener.mode, 405)
self.send_response(status)
self.end_headers()
return
self.send_response(404)
self.end_headers()
def log_message(self, *_):
return
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer):
daemon_threads = True
def mutate_json(path, function, reconcile=False):
data = json.loads(path.read_text())
function(data)
if reconcile:
recompute(data)
path.write_text(json.dumps(data, sort_keys=True))
def create_fixture(suite, name, base_url):
root = suite / name
workspace = root / "workspace-PATH_SENTINEL"
output_dir = root / "output"
raw_root = root / "raw"
workspace.mkdir(parents=True)
output_dir.mkdir()
raw_root.mkdir()
marker = root / "marker"
marker.write_text("")
descendant = root / "descendant"
descendant.write_text("")
observation = root / "observation"
observation.write_text('{"level":"info","ts":1,"msg":"node ready"}\n')
metrics = root / "metrics"
metrics.write_text("iop_anthropic_single_request_ingress_total 0\n")
claude = root / "claude"
claude.write_text(CLAUDE_FAKE)
claude.chmod(0o700)
edge = root / "edge"
edge.write_text(EDGE_FAKE)
edge.chmod(0o700)
node = root / "node"
node.write_text(NODE_FAKE)
node.chmod(0o700)
config = root / "edge-config"
config.write_text("fixed config\n")
runtime_path = root / "runtime-evidence"
model = "MODEL_SENTINEL"
output = output_dir / "manifest.json"
head = subprocess.check_output(["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], text=True).strip()
branch = subprocess.check_output(["git", "-C", str(REPO_ROOT), "rev-parse", "--abbrev-ref", "HEAD"], text=True).strip()
values = {
"runner_os": platform.system().lower(),
"runner_arch": platform.machine().lower(),
"workspace_os": platform.system().lower(),
"workspace_arch": platform.machine().lower(),
"workspace_root_digest": sha_text(str(workspace.resolve())),
"workspace_owner_digest": "",
"claude_digest": sha_file(claude),
"claude_version_digest": "sha256:" + hashlib.sha256(CLAUDE_VERSION.encode()).hexdigest(),
"claude_help_digest": "sha256:" + hashlib.sha256(CLAUDE_HELP.encode()).hexdigest(),
"edge_digest": sha_file(edge),
"edge_version_digest": "sha256:" + hashlib.sha256(EDGE_VERSION.encode()).hexdigest(),
"node_digest": sha_file(node),
"node_version_digest": "sha256:" + hashlib.sha256(NODE_VERSION.encode()).hexdigest(),
"config_digest": sha_file(config),
"config_check_digest": "sha256:" + hashlib.sha256(CONFIG_CHECK.encode()).hexdigest(),
"schema_digest": sha_file(SCHEMA),
"base_url_digest": sha_text(base_url),
"public_model_digest": sha_text(model),
"stage_engines": ["gemini", "ornith-fast", "gemini"],
"stage_binding_digest": "",
}
runtime = {
"schema_version": "1",
"source": {"head": head, "branch_digest": sha_text(branch), "worktree_digest": SELF_TEST_WORKTREE},
"runtime": values,
}
recompute(runtime)
runtime_path.write_text(json.dumps(runtime, sort_keys=True))
return {
"root": root, "workspace": workspace, "output_dir": output_dir, "raw_root": raw_root,
"descendant": descendant,
"marker": marker, "observation": observation, "metrics": metrics, "claude": claude,
"edge": edge, "node": node, "config": config, "runtime": runtime_path, "model": model,
"base_url": base_url, "metrics_url": base_url + "/metrics", "output": output,
}
def command_for(fixture, mode="--preflight-only", schema=SCHEMA, overrides=None):
values = {
"--claude": fixture["claude"], "--runtime-evidence": fixture["runtime"],
"--base-url": fixture["base_url"], "--model": fixture["model"],
"--edge-bin": fixture["edge"], "--node-bin": fixture["node"],
"--edge-config": fixture["config"],
"--observation-file": fixture["observation"], "--metrics-url": fixture["metrics_url"],
"--workspace": fixture["workspace"], "--output": fixture["output"],
"--secret-env": "IOP_SMOKE_TEST_SECRET", "--schema": schema,
}
if overrides:
values.update(overrides)
command = [SELF, mode]
for key, value in values.items():
command.extend([key, str(value)])
return command
def environment_for(fixture, extra=None):
environment = os.environ.copy()
environment.update({
"IOP_SMOKE_SELF_TEST": "1",
"IOP_SMOKE_TEST_WORKTREE_DIGEST": SELF_TEST_WORKTREE,
"IOP_SMOKE_TMP_ROOT": str(fixture["raw_root"]),
"IOP_SMOKE_TEST_SECRET": "SECRET_SENTINEL_VALUE",
"IOP_SMOKE_FAKE_MARKER": str(fixture["marker"]),
"IOP_SMOKE_FAKE_OBSERVATION": str(fixture["observation"]),
"IOP_SMOKE_FAKE_METRICS": str(fixture["metrics"]),
"IOP_SMOKE_FAKE_CONFIG": str(fixture["config"]),
"IOP_SMOKE_FAKE_NODE": str(fixture["node"]),
"IOP_SMOKE_FAKE_RUNTIME": str(fixture["runtime"]),
"IOP_SMOKE_FAKE_OUTPUT": str(fixture["output"]),
"IOP_SMOKE_FAKE_EXPECT_MODEL_DIGEST": sha_text(fixture["model"]),
"IOP_SMOKE_FAKE_EXPECT_BASE_DIGEST": sha_text(fixture["base_url"]),
"IOP_SMOKE_FAKE_EXPECT_PROMPT_DIGEST": sha_text(PROMPT),
"IOP_SMOKE_FAKE_DESCENDANT": str(fixture["descendant"]),
"CLAUDE_CODE_MAX_RETRIES": "9",
"CLAUDE_CODE_DISABLE_TERMINAL_TITLE": "0",
})
if extra:
environment.update(extra)
return environment
def assert_redacted(fixture, result, case):
combined = result.stdout + result.stderr
forbidden = [
"SECRET_SENTINEL_VALUE", fixture["model"], fixture["base_url"],
str(fixture["workspace"]), str(fixture["output"]),
]
check(all(value not in combined for value in forbidden), case + ": raw value leaked")
def assert_cleanup(fixture, case):
check(list(fixture["raw_root"].iterdir()) == [], case + ": raw temporary capture remained")
def closed_failure_reason(result):
reason = "no closed reason"
prefix = "[single-request-claude-smoke] validation failed: "
for line in result.stderr.splitlines():
if line.startswith(prefix):
candidate = line[len(prefix):]
if candidate and all(character.isalnum() or character in " ()-" for character in candidate):
reason = candidate
return reason
def run_case(fixture, case, mode="--preflight-only", expect_success=False, expect_child=0, expect_output=False, env=None, overrides=None, schema=SCHEMA):
Listener.metrics_path = fixture["metrics"]
Listener.model = fixture["model"]
result = subprocess.run(
command_for(fixture, mode=mode, schema=schema, overrides=overrides),
env=environment_for(fixture, env), capture_output=True, text=True, timeout=25,
)
assert_redacted(fixture, result, case)
if (result.returncode == 0) != expect_success:
raise TestFailure(case + ": unexpected exit status (" + closed_failure_reason(result) + ")")
marker_count = len([line for line in fixture["marker"].read_text().splitlines() if line])
if expect_child is not None and marker_count != expect_child:
raise TestFailure(case + ": unexpected child count " + str(marker_count) + " (" + closed_failure_reason(result) + ")")
assert_cleanup(fixture, case)
check(fixture["output"].exists() == expect_output, case + ": unexpected final output state")
partials = [path for path in fixture["output_dir"].iterdir() if path != fixture["output"]]
check(not partials, case + ": partial publication remained")
return result
def require_descendant_pid(fixture, case):
deadline = time.monotonic() + 10
descendant = fixture["descendant"]
while time.monotonic() < deadline:
if descendant.exists():
value = descendant.read_text().strip()
if value:
try:
return int(value)
except ValueError:
raise TestFailure(case + ": descendant PID was invalid")
time.sleep(0.05)
raise TestFailure(case + ": descendant did not start")
def assert_process_gone(pid, case):
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
try:
os.kill(pid, 0)
except ProcessLookupError:
return
time.sleep(0.05)
raise TestFailure(case + ": descendant remained")
def apply_preflight_mutation(name, fixture):
environment = {}
overrides = {}
schema = SCHEMA
Listener.mode = "ok"
if name == "support-tool":
environment["IOP_SMOKE_TEST_FAIL_CHECK"] = "support-tool"
elif name == "claude-executable":
fixture["claude"].chmod(0o600)
elif name == "edge-executable":
fixture["edge"].chmod(0o600)
elif name == "node-file":
overrides["--node-bin"] = fixture["root"] / "absent-node"
elif name == "node-executable":
fixture["node"].chmod(0o600)
elif name == "claude-version":
environment["IOP_SMOKE_FAKE_CLAUDE_VERSION"] = "fail"
elif name == "claude-help":
environment["IOP_SMOKE_FAKE_CLAUDE_HELP"] = "missing"
elif name == "edge-version":
environment["IOP_SMOKE_FAKE_EDGE_VERSION"] = "fail"
elif name == "node-version":
environment["IOP_SMOKE_FAKE_NODE_VERSION"] = "fail"
elif name == "config-check":
environment["IOP_SMOKE_FAKE_CONFIG_CHECK"] = "fail"
elif name == "runtime-file":
overrides["--runtime-evidence"] = fixture["root"] / "absent-runtime"
elif name == "source-head":
mutate_json(fixture["runtime"], lambda data: data["source"].__setitem__("head", "0" * 40))
elif name == "source-branch":
mutate_json(fixture["runtime"], lambda data: data["source"].__setitem__("branch_digest", sha_text("wrong")))
elif name == "source-worktree":
mutate_json(fixture["runtime"], lambda data: data["source"].__setitem__("worktree_digest", sha_text("wrong")))
elif name == "runner-os":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("runner_os", "wrong"))
elif name == "runner-arch":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("runner_arch", "wrong"))
elif name == "workspace-owner-os":
mutate_json(
fixture["runtime"],
lambda data: data["runtime"].__setitem__(
"workspace_os", "linux" if data["runtime"]["workspace_os"] == "darwin" else "darwin"
),
reconcile=True,
)
elif name == "workspace-owner-os-unsupported":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("workspace_os", "windows"), reconcile=True)
elif name == "workspace-root":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("workspace_root_digest", sha_text("wrong")), reconcile=True)
elif name == "claude-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("claude_digest", sha_text("wrong")))
elif name == "claude-version-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("claude_version_digest", sha_text("wrong")))
elif name == "claude-help-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("claude_help_digest", sha_text("wrong")))
elif name == "edge-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("edge_digest", sha_text("wrong")))
elif name == "edge-version-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("edge_version_digest", sha_text("wrong")))
elif name == "node-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("node_digest", sha_text("wrong")), reconcile=True)
elif name == "node-version-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("node_version_digest", sha_text("wrong")), reconcile=True)
elif name == "config-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("config_digest", sha_text("wrong")), reconcile=True)
elif name == "config-check-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("config_check_digest", sha_text("wrong")), reconcile=True)
elif name == "schema-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("schema_digest", sha_text("wrong")))
elif name == "base-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("base_url_digest", sha_text("wrong")), reconcile=True)
elif name == "model-digest":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("public_model_digest", sha_text("wrong")), reconcile=True)
elif name == "stage-engines":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("stage_engines", ["gemini", "gemini", "ornith-fast"]), reconcile=True)
elif name == "stage-binding":
mutate_json(fixture["runtime"], lambda data: data["runtime"].__setitem__("stage_binding_digest", sha_text("wrong")))
elif name == "health-listener":
Listener.mode = "health-fail"
elif name == "messages-listener":
Listener.mode = "messages-fail"
elif name == "messages-missing":
Listener.mode = "messages-missing"
elif name == "catalog-auth":
Listener.mode = "catalog-auth-reject"
elif name == "catalog-model":
Listener.mode = "catalog-model-missing"
elif name == "catalog-body":
Listener.mode = "catalog-malformed"
elif name == "catalog-ingress":
Listener.mode = "catalog-ingress-change"
elif name == "metrics-listener":
Listener.mode = "metrics-fail"
elif name == "metrics-counter":
fixture["metrics"].write_text("unrelated 1\n")
elif name == "observation-log":
fixture["observation"].unlink()
fixture["observation"].mkdir()
elif name == "observation-plain":
fixture["observation"].write_text("process stdout without structured Edge events\n")
elif name == "workspace":
overrides["--workspace"] = fixture["root"] / "absent-workspace"
elif name == "secret-name":
overrides["--secret-env"] = "bad-name"
elif name == "secret-value":
overrides["--secret-env"] = "IOP_SMOKE_ABSENT_SECRET"
elif name == "output-existing":
fixture["output"].write_text("occupied\n")
elif name == "result-existing":
(fixture["workspace"] / "smoke-result.txt").write_text(EXPECTED_RESULT + "\n")
elif name == "schema-contract":
bad_schema = fixture["root"] / "bad-schema"
data = json.loads(SCHEMA.read_text())
data["$defs"]["workspace"].pop("additionalProperties")
bad_schema.write_text(json.dumps(data))
schema = bad_schema
else:
raise TestFailure(name + ": unknown preflight mutation")
return environment, overrides, schema
try:
with tempfile.TemporaryDirectory(prefix="single-request-claude-self-test.", dir=REPO_ROOT / "build") as suite_raw:
suite = pathlib.Path(suite_raw)
server = Server(("127.0.0.1", 0), Listener)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
base_url = "http://127.0.0.1:" + str(server.server_address[1])
try:
double_messages_request = urllib.request.Request(base_url + "/v1/v1/messages", method="OPTIONS")
try:
urllib.request.urlopen(double_messages_request, timeout=5)
raise TestFailure("exact-route: double-v1 Messages path was accepted")
except urllib.error.HTTPError as error:
check(error.code == 404, "exact-route: double-v1 Messages path did not return 404")
positive_preflight = create_fixture(suite, "positive-preflight", base_url)
Listener.mode = "ok"
run_case(positive_preflight, "positive-preflight", expect_success=True)
authenticated_preflight = create_fixture(suite, "authenticated-preflight", base_url)
Listener.mode = "messages-auth"
run_case(authenticated_preflight, "authenticated-preflight", expect_success=True)
Listener.mode = "ok"
terminal_v1_preflight = create_fixture(suite, "terminal-v1-preflight", base_url + "/v1")
run_case(terminal_v1_preflight, "terminal-v1-preflight")
preflight_cases = [
"support-tool", "claude-executable", "edge-executable", "node-file", "node-executable",
"claude-version", "claude-help", "edge-version", "node-version", "config-check", "runtime-file",
"source-head", "source-branch", "source-worktree", "runner-os", "runner-arch",
"workspace-owner-os", "workspace-owner-os-unsupported", "workspace-root", "claude-digest",
"claude-version-digest", "claude-help-digest", "edge-digest", "edge-version-digest",
"node-digest", "node-version-digest", "config-digest", "config-check-digest",
"schema-digest", "base-digest", "model-digest",
"stage-engines", "stage-binding", "health-listener", "messages-listener", "messages-missing", "metrics-listener",
"catalog-auth", "catalog-model", "catalog-body", "catalog-ingress",
"metrics-counter", "observation-log", "observation-plain", "workspace", "secret-name", "secret-value",
"output-existing", "result-existing", "schema-contract",
]
for name in preflight_cases:
fixture = create_fixture(suite, "preflight-" + name, base_url)
environment, overrides, selected_schema = apply_preflight_mutation(name, fixture)
run_case(
fixture,
"preflight-" + name,
env=environment,
overrides=overrides,
schema=selected_schema,
expect_output=name == "output-existing",
)
Listener.mode = "ok"
valid = create_fixture(suite, "run-valid", base_url)
Listener.mode = "ok"
run_case(valid, "run-valid", mode="--run", expect_success=True, expect_child=1, expect_output=True)
subprocess.run([SELF, "--validate-manifest", str(valid["output"]), "--schema", str(SCHEMA)], check=True, capture_output=True, text=True, timeout=10)
manifest = json.loads(valid["output"].read_text())
check(manifest["workspace"]["changed"] is True, "run-valid: workspace change missing")
check(manifest["verification"]["exit_code"] == 0, "run-valid: verifier result missing")
check(manifest["runtime"]["public_model_digest"] == sha_text(valid["model"]), "run-valid: model binding missing")
check(manifest["runtime"]["node_digest"] == sha_file(valid["node"]), "run-valid: Node binding missing")
check(manifest["runtime"]["node_version_digest"] == sha_text(NODE_VERSION), "run-valid: Node version binding missing")
check(manifest["terminal"]["duration_ms"] == 11, "run-valid: terminal total missing")
mutations = {
"extra-key": lambda data: data.__setitem__("prompt", "SECRET_SENTINEL"),
"source": lambda data: data["source"].__setitem__("branch_digest", "bad"),
"base": lambda data: data["runtime"].__setitem__("base_url_digest", sha_text("changed")),
"model": lambda data: data["runtime"].__setitem__("public_model_digest", sha_text("changed")),
"config": lambda data: data["runtime"].__setitem__("config_digest", sha_text("changed")),
"node": lambda data: data["runtime"].__setitem__("node_digest", sha_text("changed")),
"node-version": lambda data: data["runtime"].__setitem__("node_version_digest", sha_text("changed")),
"binding": lambda data: data["runtime"].__setitem__("stage_binding_digest", sha_text("changed")),
"engine": lambda data: data["stages"][1].__setitem__("engine_family", "gemini"),
"stage": lambda data: data["stages"][0].__setitem__("stage", "work"),
"terminal": lambda data: data["terminal"].__setitem__("count", 2),
"ingress-boolean": lambda data: data["ingress"].__setitem__("delta", True),
"terminal-boolean": lambda data: data["terminal"].__setitem__("count", True),
"workspace-change": lambda data: data["workspace"].__setitem__("changed", False),
"workspace-digest": lambda data: data["workspace"].__setitem__("after_digest", data["workspace"]["before_digest"]),
"verifier": lambda data: data["verification"].__setitem__("exit_code", 1),
"verifier-boolean": lambda data: data["verification"].__setitem__("exit_code", False),
"redaction-boolean": lambda data: data["redaction"].__setitem__("forbidden_match_count", False),
}
for name, mutation in mutations.items():
candidate = copy.deepcopy(manifest)
mutation(candidate)
path = valid["root"] / ("manifest-" + name)
path.write_text(json.dumps(candidate))
result = subprocess.run([SELF, "--validate-manifest", str(path), "--schema", str(SCHEMA)], capture_output=True, text=True, timeout=10)
check(result.returncode != 0, "manifest mutation accepted: " + name)
classified_failures = [
("cli", "claude-failure-cli", "cli-validation", "cli-usage"),
("auth", "claude-failure-auth", "authentication-rejected", "authentication"),
("transport", "claude-failure-transport", "transport-failure", "connection-refused"),
("connection", "claude-failure-connection", "transport-failure", "connection-error"),
("tls", "claude-failure-tls", "transport-failure", "tls-certificate"),
("api", "claude-failure-api", "api-rejected", "http-429"),
("api-400", "claude-failure-api-400", "api-rejected", "http-400"),
("api-400-beta", "claude-failure-api-400-beta", "api-rejected", "unsupported-beta"),
("api-400-field", "claude-failure-api-400-field", "api-rejected", "unknown-field"),
("api-400-thinking", "claude-failure-api-400-thinking", "api-rejected", "invalid-thinking"),
("api-400-output", "claude-failure-api-400-output", "api-rejected", "invalid-output-config"),
("unknown", "claude-failure-unknown", "unknown", "unclassified"),
]
for name, behavior, failure_class, failure_reason in classified_failures:
fixture = create_fixture(suite, "run-classified-" + name, base_url)
result = run_case(
fixture, "run-classified-" + name, mode="--run", expect_child=1,
env={"IOP_SMOKE_FAKE_BEHAVIOR": behavior},
)
check("class " + failure_class + " reason " + failure_reason + ")" in result.stderr, "run-classified-" + name + ": closed diagnostic missing")
for marker in ["rejected-field-marker", "secret_rejected_beta", "secret_field_marker", "secret_thinking_marker", "secret_output_marker", "secret_connection_marker", "secret_cert_marker"]:
check(marker not in result.stderr.lower(), "run-classified-" + name + ": raw diagnostic text leaked")
run_failures = [
("stale", "stale", {}, False),
("rotated", "rotated", {}, False),
("no-change", "no-change", {}, False),
("wrong-content", "wrong-content", {}, False),
("verifier-failure", "success", {"IOP_SMOKE_TEST_FORCE_VERIFIER_FAILURE": "1"}, False),
("config-after", "config-after", {}, False),
("node-after", "node-after", {}, False),
("runtime-after", "runtime-after", {}, False),
("timing-swapped", "timing-swapped", {}, False),
("terminal-no-result", "terminal-no-result", {}, False),
("terminal-false-result", "terminal-false-result", {}, False),
("output-race", "output-race", {}, True),
]
for name, behavior, extra, expect_output in run_failures:
fixture = create_fixture(suite, "run-" + name, base_url)
environment = {"IOP_SMOKE_FAKE_BEHAVIOR": behavior, **extra}
run_case(fixture, "run-" + name, mode="--run", expect_child=1, expect_output=expect_output, env=environment)
if name == "output-race":
check(fixture["output"].read_text() == "concurrent owner\n", "run-output-race: existing target was overwritten")
leader_exit_fixture = create_fixture(suite, "run-leader-exit", base_url)
started = time.monotonic()
run_case(
leader_exit_fixture,
"run-leader-exit",
mode="--run",
expect_child=1,
env={"IOP_SMOKE_FAKE_BEHAVIOR": "leader-exit"},
)
check(time.monotonic() - started < 6, "run-leader-exit: supervisor cleanup was not bounded")
assert_process_gone(require_descendant_pid(leader_exit_fixture, "run-leader-exit"), "run-leader-exit")
early_signal_fixture = create_fixture(suite, "run-early-signal", base_url)
started = time.monotonic()
run_case(
early_signal_fixture,
"run-early-signal",
mode="--run",
expect_child=None,
env={
"IOP_SMOKE_FAKE_BEHAVIOR": "early-signal",
"IOP_CLAUDE_SUPERVISOR_TEST_EARLY_SIGNAL": "1",
},
)
check(time.monotonic() - started < 6, "run-early-signal: supervisor cleanup was not bounded")
early_descendant = early_signal_fixture["descendant"]
if early_descendant.exists() and early_descendant.read_text().strip():
assert_process_gone(require_descendant_pid(early_signal_fixture, "run-early-signal"), "run-early-signal")
signal_fixture = create_fixture(suite, "run-term-resistant", base_url)
Listener.metrics_path = signal_fixture["metrics"]
process = subprocess.Popen(
command_for(signal_fixture, mode="--run"),
env=environment_for(signal_fixture, {"IOP_SMOKE_FAKE_BEHAVIOR": "term-resistant"}),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
)
deadline = time.monotonic() + 10
while time.monotonic() < deadline and not signal_fixture["marker"].read_text().strip():
time.sleep(0.05)
check(bool(signal_fixture["marker"].read_text().strip()), "run-signal: child did not start")
descendant_pid = require_descendant_pid(signal_fixture, "run-term-resistant")
process.terminate()
started = time.monotonic()
stdout, stderr = process.communicate(timeout=10)
check(time.monotonic() - started < 6, "run-term-resistant: supervisor cleanup was not bounded")
check(process.returncode != 0, "run-term-resistant: interruption succeeded unexpectedly")
class Result:
pass
result = Result()
result.stdout, result.stderr = stdout, stderr
assert_redacted(signal_fixture, result, "run-term-resistant")
assert_cleanup(signal_fixture, "run-term-resistant")
check(not signal_fixture["output"].exists(), "run-term-resistant: final output remained")
check(list(signal_fixture["output_dir"].iterdir()) == [], "run-term-resistant: partial publication remained")
assert_process_gone(descendant_pid, "run-term-resistant")
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
except TestFailure as error:
print("[single-request-claude-self-test] " + str(error), file=sys.stderr)
raise SystemExit(1)
except Exception:
print("[single-request-claude-self-test] unexpected self-test failure", file=sys.stderr)
raise SystemExit(1)
PY
log 'self-test passed: exact Claude base-route coverage, structured observation admission, child-only zero retry, authenticated model admission, closed failure classification, model/Edge/Node/runtime binding, zero-child preflight, derived verification, redaction, cleanup, signal handling, and atomic publication'
}
main() {
parse_args "$@"
case "$MODE" in
self-test)
self_test
;;
validate-manifest)
[ -f "$MANIFEST" ] && [ -f "$SCHEMA" ] || fail 'manifest or schema unavailable'
validate_schema_contract "$SCHEMA" 2>/dev/null || fail 'manifest schema invalid'
validate_manifest "$MANIFEST" "$SCHEMA" || fail 'manifest validation failed'
log 'manifest is valid'
;;
preflight-only)
preflight_only
;;
run)
run_once
;;
*)
usage
exit "$EXIT_USAGE"
;;
esac
}
main "$@"
Reference in a new issue View git blame Copy permalink