toki
8a4f6c55a1
- Update roadmap milestones and phase docs across multiple phases - Update plan, code-review, create-roadmap, update-roadmap, finalize-task-routing skills - Update dev-corp-runtime-deploy, dev-runtime-deploy, orchestrate-agent-task-loop skills - Refactor agent-task-loop dispatch script - Add streamgate Go package (commit_boundary, evidence_tail, filter_registry, stream_release) - Add test inventory files (dev, dev-corp, unified) - Update test smoke tests and rules for dev/dev-corp - Update docs/edge-local-dev-guide and e2e scripts - Update inventory-query Go package - Remove deprecated templates and inventory.yaml files - Add orchestrate-agent-task-loop tests
584 lines
25 KiB
Bash
Executable file
584 lines
25 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Long-context admission dev capacity smoke.
|
|
#
|
|
# Milestone: model-group-long-context-admission, SDD S09 (capacity-smoke).
|
|
# Verifies, against the live dev provider pool (GX10 vLLM, OneXPlayer Lemonade,
|
|
# mac-mlx-vllm), that:
|
|
# normal-10 - normal capacity+1 concurrency reaches total in_flight and
|
|
# queues at least one request, then recovers to zero.
|
|
# mixed - long requests hold long slots while normal requests still
|
|
# dispatch without head-of-line blocking, then recover.
|
|
# all-long-slot-full - more long requests than total long slots queue, then
|
|
# dispatch as long slots free up, then recover to zero.
|
|
#
|
|
# This is a live-dependency auxiliary smoke. It does not stand up any process; it
|
|
# drives an already-running dev Edge OpenAI-compatible endpoint and reads the
|
|
# Control Plane status snapshot. Every reachability failure is recorded as an
|
|
# exact failed command (a verification blocker), never turned into a user-review
|
|
# request.
|
|
#
|
|
# Secrets: the bearer token is read from IOP_LONG_SMOKE_TOKEN only and is never
|
|
# printed to stdout/stderr or written to the out-dir.
|
|
#
|
|
# Options:
|
|
# --preflight run source/config/endpoint/identity/snapshot checks only
|
|
# --scenario <name> normal-10 | mixed | all-long-slot-full
|
|
# --config <path> edge config for `edge config check` (default configs/edge.yaml)
|
|
# --out-dir <dir> evidence output dir (default /tmp/iop-long-admission-smoke)
|
|
# -h | --help usage
|
|
#
|
|
# Environment (defaults target the dev-runtime inventory; override per host):
|
|
# IOP_LONG_SMOKE_BASE_URL Edge OpenAI-compatible base URL (default http://toki-labs.com:18083/v1)
|
|
# IOP_LONG_SMOKE_STATUS_URL Control Plane status URL (default runner-local edge status)
|
|
# IOP_LONG_SMOKE_STATUS_SSH optional ssh target used to curl STATUS_URL from a host
|
|
# that cannot reach it directly (e.g. toki@toki-labs.com)
|
|
# IOP_LONG_SMOKE_TOKEN optional bearer token for the OpenAI-compatible endpoint
|
|
# IOP_LONG_SMOKE_MODEL model alias (default qwen3.6:35b)
|
|
# IOP_LONG_SMOKE_LONG_CHARS synthetic long-prompt size in chars (default 720000 ~ 200k tokens)
|
|
# IOP_LONG_SMOKE_MAX_TOKENS completion max_tokens per request (default 64)
|
|
# IOP_LONG_SMOKE_POLL_SECS hard ceiling seconds for the status poll loop; the
|
|
# loop otherwise runs until requests finish (default 3600)
|
|
# IOP_LONG_SMOKE_EDGE_BIN optional prebuilt edge binary used for `config check`
|
|
# instead of `go run` (for runners without a go toolchain)
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# REPO_ROOT is where `edge config check` runs. Defaults to the script's parent
|
|
# repo, but can be overridden (e.g. when the script is copied to /tmp on a runner
|
|
# whose checkout lives elsewhere).
|
|
REPO_ROOT="${IOP_LONG_SMOKE_REPO:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
|
|
|
BASE_URL="${IOP_LONG_SMOKE_BASE_URL:-http://toki-labs.com:18083/v1}"
|
|
STATUS_URL="${IOP_LONG_SMOKE_STATUS_URL:-http://127.0.0.1:18001/edges/edge-toki-labs-dev/status}"
|
|
STATUS_SSH="${IOP_LONG_SMOKE_STATUS_SSH:-}"
|
|
MODEL="${IOP_LONG_SMOKE_MODEL:-qwen3.6:35b}"
|
|
LONG_CHARS="${IOP_LONG_SMOKE_LONG_CHARS:-720000}"
|
|
MAX_TOKENS="${IOP_LONG_SMOKE_MAX_TOKENS:-64}"
|
|
POLL_SECS="${IOP_LONG_SMOKE_POLL_SECS:-3600}"
|
|
EDGE_BIN="${IOP_LONG_SMOKE_EDGE_BIN:-}"
|
|
|
|
CONFIG="configs/edge.yaml"
|
|
OUT_DIR="/tmp/iop-long-admission-smoke"
|
|
SCENARIO=""
|
|
DO_PREFLIGHT=0
|
|
|
|
# dev pool baseline (agent-test/inventory-dev.yaml).
|
|
PROVIDER_IDS=(gx10-vllm onexplayer-lemonade mac-mlx-vllm)
|
|
NORMAL_CAPACITY_TOTAL=9 # gx10=4 + onexplayer=3 + mac=2
|
|
LONG_SLOT_TOTAL=4 # gx10=1 + onexplayer=2 + mac=1
|
|
|
|
usage() {
|
|
sed -n '3,54p' "$0"
|
|
}
|
|
|
|
log() { echo "[long-admission-smoke] $*"; }
|
|
|
|
die() {
|
|
log "ERROR: $*"
|
|
exit 1
|
|
}
|
|
|
|
# ── argument parsing ─────────────────────────────────────────────────────────
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--preflight) DO_PREFLIGHT=1; shift ;;
|
|
--scenario) SCENARIO="${2:?--scenario requires a value}"; shift 2 ;;
|
|
--config) CONFIG="${2:?--config requires a value}"; shift 2 ;;
|
|
--out-dir) OUT_DIR="${2:?--out-dir requires a value}"; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) die "unknown argument: $1 (see --help)" ;;
|
|
esac
|
|
done
|
|
|
|
if [ "$DO_PREFLIGHT" -eq 0 ] && [ -z "$SCENARIO" ]; then
|
|
die "nothing to do: pass --preflight and/or --scenario <name>"
|
|
fi
|
|
|
|
command -v curl >/dev/null 2>&1 || die "curl is required"
|
|
command -v jq >/dev/null 2>&1 || die "jq is required"
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
RUN_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
log "out-dir=$OUT_DIR run=$RUN_STAMP base_url=$BASE_URL"
|
|
|
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
|
|
|
# curl auth args, built once. Expanded with the ${arr[@]+...} idiom so an empty
|
|
# array survives `set -u` on the dev runner's macOS bash 3.2, which also lacks
|
|
# `mapfile` (do not reintroduce bash-4-only builtins here).
|
|
AUTH_ARGS=()
|
|
if [ -n "${IOP_LONG_SMOKE_TOKEN:-}" ]; then
|
|
AUTH_ARGS=(-H "Authorization: Bearer ${IOP_LONG_SMOKE_TOKEN}")
|
|
fi
|
|
|
|
# fetch_status <out-file>: capture the Control Plane status JSON.
|
|
fetch_status() {
|
|
local out="$1"
|
|
if [ -n "$STATUS_SSH" ]; then
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 "$STATUS_SSH" \
|
|
"curl -fsS --connect-timeout 10 '$STATUS_URL'" >"$out" 2>"$out.err"
|
|
else
|
|
curl -fsS --connect-timeout 10 "$STATUS_URL" >"$out" 2>"$out.err"
|
|
fi
|
|
}
|
|
|
|
# provider_snapshots_jq: normalize provider snapshots from either the edge status
|
|
# response shape or a fleet node list into a flat array.
|
|
SNAP_JQ='[.. | objects | select(has("provider_snapshots")) | .provider_snapshots[]?]'
|
|
|
|
# summarize_snapshot <status-json> -> prints a compact per-provider line and totals.
|
|
summarize_snapshot() {
|
|
local file="$1"
|
|
jq -r "
|
|
${SNAP_JQ} as \$s
|
|
| (\$s | map(.in_flight // 0) | add // 0) as \$inflight
|
|
| (\$s | map(.queued // 0) | add // 0) as \$queued
|
|
| (\$s | map(.long_in_flight // 0) | add // 0) as \$loninflight
|
|
| (\$s | map(.long_queued // 0) | add // 0) as \$lonqueued
|
|
| (\$s | map(has(\"long_in_flight\")) | any) as \$haslong
|
|
| (\$s[] | \" provider=\(.id // .adapter // \"?\") capacity=\(.capacity // 0) in_flight=\(.in_flight // 0) queued=\(.queued // 0) long_in_flight=\(.long_in_flight // \"n/a\") long_queued=\(.long_queued // \"n/a\") health=\(.health // \"?\")\"),
|
|
\" TOTAL in_flight=\(\$inflight) queued=\(\$queued) long_in_flight=\(if \$haslong then \$loninflight else \"n/a\" end) long_queued=\(if \$haslong then \$lonqueued else \"n/a\" end)\"
|
|
" "$file" 2>/dev/null || echo " (unparseable status json)"
|
|
}
|
|
|
|
# totals_from_snapshot <status-json> <key> -> integer sum, or empty when key absent everywhere.
|
|
totals_from_snapshot() {
|
|
local file="$1" key="$2"
|
|
jq -r "${SNAP_JQ} as \$s
|
|
| if (\$s | map(has(\"${key}\")) | any) then (\$s | map(.${key} // 0) | add // 0) else \"\" end" \
|
|
"$file" 2>/dev/null
|
|
}
|
|
|
|
# build the synthetic long prompt file once (kept in out-dir, not repo).
|
|
LONG_PROMPT_FILE="$OUT_DIR/long_prompt_${RUN_STAMP}.txt"
|
|
ensure_long_prompt() {
|
|
[ -f "$LONG_PROMPT_FILE" ] && return 0
|
|
# Repeat a filler sentence to LONG_CHARS. Synthetic text is enough; a
|
|
# tokenizer-perfect fixture is intentionally out of scope.
|
|
local unit="Long context admission smoke synthetic filler segment. "
|
|
local buf=""
|
|
while [ "${#buf}" -lt "$LONG_CHARS" ]; do
|
|
buf="${buf}${unit}"
|
|
done
|
|
printf '%s' "${buf:0:$LONG_CHARS}" >"$LONG_PROMPT_FILE"
|
|
log "synthetic long prompt: ${LONG_CHARS} chars -> $LONG_PROMPT_FILE"
|
|
}
|
|
|
|
# build a chat completion request body file for a given kind (normal|long).
|
|
# $1=kind $2=idx $3=out-body-file
|
|
build_body() {
|
|
local kind="$1" idx="$2" out="$3" content
|
|
if [ "$kind" = "long" ]; then
|
|
ensure_long_prompt
|
|
content="$(cat "$LONG_PROMPT_FILE")
|
|
Summarize the above in one short sentence. request_idx=${idx}"
|
|
else
|
|
content="Reply with the single word OK. request_idx=${idx}"
|
|
fi
|
|
jq -n --arg model "$MODEL" --arg content "$content" --argjson maxt "$MAX_TOKENS" \
|
|
'{model:$model, max_tokens:$maxt, stream:false, messages:[{role:"user", content:$content}]}' \
|
|
>"$out"
|
|
}
|
|
|
|
# fire one request in the background; write HTTP code + body to out prefix.
|
|
# $1=kind $2=idx $3=out-prefix
|
|
fire_request() {
|
|
local kind="$1" idx="$2" prefix="$3"
|
|
local body="$prefix.req.json"
|
|
build_body "$kind" "$idx" "$body"
|
|
curl -sS -o "$prefix.resp.json" -w '%{http_code}' \
|
|
--connect-timeout 15 --max-time 1800 \
|
|
-H "Content-Type: application/json" \
|
|
${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \
|
|
-X POST "$BASE_URL/chat/completions" \
|
|
--data-binary "@$body" >"$prefix.code" 2>"$prefix.curl.err" || true
|
|
}
|
|
|
|
# poll status repeatedly into a peaks file while a scenario runs. The poll loop
|
|
# runs until the caller creates "$OUT_DIR/<label>.polling_done" (so peaks are
|
|
# captured for the full request duration, however slow), bounded by POLL_SECS as
|
|
# a hard safety ceiling.
|
|
# $1=label $2=peaks-out-file
|
|
poll_peaks() {
|
|
local label="$1" peaks="$2"
|
|
local deadline=$((SECONDS + POLL_SECS))
|
|
local doneflag="$OUT_DIR/${label}.polling_done"
|
|
local snap="$OUT_DIR/${label}_poll_${RUN_STAMP}.json"
|
|
local peak_inflight=0 peak_queued=0 peak_loninflight=0 peak_lonqueued=0 samples=0
|
|
local long_visible=0
|
|
rm -f "$doneflag"
|
|
while (( SECONDS < deadline )) && [ ! -f "$doneflag" ]; do
|
|
if fetch_status "$snap"; then
|
|
samples=$((samples + 1))
|
|
local inflight queued lon lonq
|
|
inflight="$(totals_from_snapshot "$snap" in_flight)"; inflight="${inflight:-0}"
|
|
queued="$(totals_from_snapshot "$snap" queued)"; queued="${queued:-0}"
|
|
lon="$(totals_from_snapshot "$snap" long_in_flight)"
|
|
lonq="$(totals_from_snapshot "$snap" long_queued)"
|
|
[ -n "$lon" ] && long_visible=1
|
|
(( inflight > peak_inflight )) && peak_inflight=$inflight
|
|
(( queued > peak_queued )) && peak_queued=$queued
|
|
[ -n "$lon" ] && (( lon > peak_loninflight )) && peak_loninflight=$lon
|
|
[ -n "$lonq" ] && (( lonq > peak_lonqueued )) && peak_lonqueued=$lonq
|
|
fi
|
|
sleep 3
|
|
done
|
|
{
|
|
echo "label=$label samples=$samples"
|
|
echo "peak_in_flight=$peak_inflight"
|
|
echo "peak_queued=$peak_queued"
|
|
if [ "$long_visible" -eq 1 ]; then
|
|
echo "peak_long_in_flight=$peak_loninflight"
|
|
echo "peak_long_queued=$peak_lonqueued"
|
|
else
|
|
echo "peak_long_in_flight=n/a (Control Plane status view does not expose long fields)"
|
|
echo "peak_long_queued=n/a (Control Plane status view does not expose long fields)"
|
|
fi
|
|
} >"$peaks"
|
|
cat "$peaks"
|
|
}
|
|
|
|
# capture a final snapshot and assert recovery to zero.
|
|
# $1=label -> returns non-zero when recovery assertion fails.
|
|
assert_recovery() {
|
|
local label="$1"
|
|
local snap="$OUT_DIR/${label}_final_${RUN_STAMP}.json"
|
|
local ok=1
|
|
if ! fetch_status "$snap"; then
|
|
log "$label: FAILED to fetch final status (see $snap.err)"
|
|
return 1
|
|
fi
|
|
log "$label final snapshot:"
|
|
summarize_snapshot "$snap"
|
|
local inflight queued lon lonq
|
|
inflight="$(totals_from_snapshot "$snap" in_flight)"; inflight="${inflight:-0}"
|
|
queued="$(totals_from_snapshot "$snap" queued)"; queued="${queued:-0}"
|
|
lon="$(totals_from_snapshot "$snap" long_in_flight)"
|
|
lonq="$(totals_from_snapshot "$snap" long_queued)"
|
|
[ "$inflight" = "0" ] || { log "$label: in_flight not recovered ($inflight)"; ok=0; }
|
|
[ "$queued" = "0" ] || { log "$label: queued not recovered ($queued)"; ok=0; }
|
|
if [ -n "$lon" ]; then
|
|
[ "$lon" = "0" ] || { log "$label: long_in_flight not recovered ($lon)"; ok=0; }
|
|
[ "${lonq:-0}" = "0" ] || { log "$label: long_queued not recovered ($lonq)"; ok=0; }
|
|
else
|
|
log "$label: long_in_flight/long_queued not exposed by status view; verify recovery via Edge dispatch logs (context_class/queue_reason)."
|
|
fi
|
|
[ "$ok" -eq 1 ] || return 1
|
|
log "$label: recovery OK (in_flight=0, queued=0)"
|
|
return 0
|
|
}
|
|
|
|
# read_kv <kv-file> <key>: print the value after "key=" for the first matching
|
|
# line. The value may be a number or an "n/a ..." note.
|
|
read_kv() {
|
|
local file="$1" key="$2"
|
|
sed -n "s/^${key}=//p" "$file" 2>/dev/null | head -1
|
|
}
|
|
|
|
# assert_peak <peaks-file> <key> <op> <bound> <label>: assert a numeric peak
|
|
# field satisfies a comparison. <op> is a test(1) integer operator (-ge/-le/...).
|
|
# Returns non-zero when the field is missing, non-numeric, or fails the bound.
|
|
assert_peak() {
|
|
local file="$1" key="$2" op="$3" bound="$4" label="$5" val
|
|
val="$(read_kv "$file" "$key")"
|
|
if ! [[ "$val" =~ ^[0-9]+$ ]]; then
|
|
log "$label: FAIL peak $key not a number ('${val:-<absent>}'); cannot assert $key $op $bound"
|
|
return 1
|
|
fi
|
|
if [ "$val" "$op" "$bound" ]; then
|
|
log "$label: peak $key=$val ($op $bound) OK"
|
|
return 0
|
|
fi
|
|
log "$label: FAIL peak $key=$val (require $key $op $bound)"
|
|
return 1
|
|
}
|
|
|
|
# fire N requests of a kind, wait for all, print success/http summary.
|
|
# $1=kind $2=count $3=label
|
|
fire_and_wait() {
|
|
local kind="$1" count="$2" label="$3"
|
|
local pids=() i
|
|
log "$label: firing $count $kind request(s) to $BASE_URL/chat/completions"
|
|
for ((i = 1; i <= count; i++)); do
|
|
fire_request "$kind" "$i" "$OUT_DIR/${label}_${kind}_${i}_${RUN_STAMP}" &
|
|
pids+=("$!")
|
|
done
|
|
# poll while they run.
|
|
poll_peaks "$label" "$OUT_DIR/${label}_peaks_${RUN_STAMP}.txt" &
|
|
local poll_pid=$!
|
|
for p in "${pids[@]}"; do wait "$p" 2>/dev/null || true; done
|
|
touch "$OUT_DIR/${label}.polling_done"
|
|
wait "$poll_pid" 2>/dev/null || true
|
|
local ok=0 total=0
|
|
for ((i = 1; i <= count; i++)); do
|
|
total=$((total + 1))
|
|
local code
|
|
code="$(cat "$OUT_DIR/${label}_${kind}_${i}_${RUN_STAMP}.code" 2>/dev/null || echo "000")"
|
|
[ "$code" = "200" ] && ok=$((ok + 1))
|
|
done
|
|
log "$label: $kind http_200=$ok/$total"
|
|
if [ "$ok" -ne "$total" ]; then
|
|
log "$label: FAIL $kind http_200=$ok/$total (require $total/$total)"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ── preflight helpers ────────────────────────────────────────────────────────
|
|
|
|
# preflight_check_tools
|
|
# Verifies required tools (curl, jq) and optional edge binary availability.
|
|
# Returns 0 if all required tools are present.
|
|
preflight_check_tools() {
|
|
local ok=1
|
|
command -v curl >/dev/null 2>&1 || { log " BLOCKER: curl is required"; ok=0; }
|
|
command -v jq >/dev/null 2>&1 || { log " BLOCKER: jq is required"; ok=0; }
|
|
if [ -n "$EDGE_BIN" ]; then
|
|
if [ ! -x "$EDGE_BIN" ]; then
|
|
log " BLOCKER: edge binary not executable: $EDGE_BIN"
|
|
ok=0
|
|
else
|
|
log " edge binary available: $EDGE_BIN"
|
|
fi
|
|
else
|
|
command -v go >/dev/null 2>&1 || { log " NOTE: go not found; prebuilt EDGE_BIN required"; ok=0; }
|
|
fi
|
|
return $((1 - ok))
|
|
}
|
|
|
|
# preflight_check_config <out-file>
|
|
# Runs `edge config check` and records output. Uses prebuilt EDGE_BIN if set.
|
|
# Returns 0 on success; records rc and output in out-file.
|
|
preflight_check_config() {
|
|
local out_file="$1"
|
|
local -a cfg_cmd
|
|
if [ -n "$EDGE_BIN" ]; then
|
|
cfg_cmd=("$EDGE_BIN" config check --config "$CONFIG")
|
|
else
|
|
cfg_cmd=(go run ./apps/edge/cmd/edge config check --config "$CONFIG")
|
|
fi
|
|
local cfg_out cfg_rc=0
|
|
cfg_out="$(cd "$REPO_ROOT" && "${cfg_cmd[@]}" 2>&1)" || cfg_rc=$?
|
|
{
|
|
echo "run=$RUN_STAMP"
|
|
echo "workdir=$REPO_ROOT"
|
|
echo "base_url=$BASE_URL"
|
|
echo "status_url=$STATUS_URL status_ssh=${STATUS_SSH:-<direct>}"
|
|
echo "config=$CONFIG"
|
|
echo
|
|
echo "## source state"
|
|
echo "\$ git -C $REPO_ROOT rev-parse HEAD"
|
|
git -C "$REPO_ROOT" rev-parse HEAD 2>&1 || true
|
|
echo "\$ git -C $REPO_ROOT status --short"
|
|
git -C "$REPO_ROOT" status --short 2>&1 || true
|
|
echo
|
|
echo "## config check"
|
|
echo "\$ ${cfg_cmd[*]}"
|
|
echo "$cfg_out"
|
|
if [ "$cfg_rc" -eq 0 ]; then echo "config check OK"; else echo "config check FAILED (rc=$cfg_rc)"; fi
|
|
} | tee "$out_file"
|
|
return "$cfg_rc"
|
|
}
|
|
|
|
# preflight_check_endpoint <out-file>
|
|
# Verifies /models endpoint reachability and lists available models.
|
|
# Returns 0 on success.
|
|
preflight_check_endpoint() {
|
|
local out_file="$1"
|
|
log "endpoint reachability: $BASE_URL/models"
|
|
if curl -fsS --connect-timeout 10 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} "$BASE_URL/models" \
|
|
>"$out_file" 2>"$out_file.err"; then
|
|
log " /models OK"
|
|
jq -r '.data[]?.id' "$out_file" 2>/dev/null | sed 's/^/ model=/' || true
|
|
return 0
|
|
else
|
|
log " BLOCKER: /models unreachable. exact command:"
|
|
log " curl -fsS --connect-timeout 10 $BASE_URL/models"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# preflight_check_status <out-file>
|
|
# Verifies status endpoint reachability, provider identity/health, and capacity.
|
|
# Returns 0 on success.
|
|
preflight_check_status() {
|
|
local out_file="$1"
|
|
log "status reachability: $STATUS_URL"
|
|
if fetch_status "$out_file"; then
|
|
log " status OK -> $out_file"
|
|
summarize_snapshot "$out_file"
|
|
|
|
# every expected provider identity must be present and healthy.
|
|
local pid phealth ok=1
|
|
for pid in "${PROVIDER_IDS[@]}"; do
|
|
if ! jq -e "${SNAP_JQ} | map(.id // .adapter) | index(\"$pid\")" "$out_file" >/dev/null 2>&1; then
|
|
log " BLOCKER: provider identity not found in status snapshot: $pid"
|
|
ok=0
|
|
continue
|
|
fi
|
|
phealth="$(jq -r "${SNAP_JQ} | map(select((.id // .adapter) == \"$pid\")) | (.[0].health // \"?\")" "$out_file" 2>/dev/null || echo "?")"
|
|
if [ "$phealth" = "healthy" ]; then
|
|
log " provider present+healthy: $pid"
|
|
else
|
|
log " BLOCKER: provider not healthy: $pid (health=$phealth)"
|
|
ok=0
|
|
fi
|
|
done
|
|
|
|
# summed provider capacity must match the documented normal baseline.
|
|
local cap_total
|
|
cap_total="$(totals_from_snapshot "$out_file" capacity)"
|
|
if [ -n "$cap_total" ] && [ "$cap_total" = "$NORMAL_CAPACITY_TOTAL" ]; then
|
|
log " capacity total OK: $cap_total (== $NORMAL_CAPACITY_TOTAL)"
|
|
else
|
|
log " BLOCKER: capacity total '${cap_total:-<absent>}' != expected $NORMAL_CAPACITY_TOTAL"
|
|
ok=0
|
|
fi
|
|
|
|
if [ -z "$(totals_from_snapshot "$out_file" long_in_flight)" ]; then
|
|
log " NOTE: status view does not expose long_in_flight/long_queued; long-slot recovery must be read from Edge dispatch logs (context_class/queue_reason) or a follow-up that adds long fields to the Control Plane status view."
|
|
fi
|
|
|
|
[ "$ok" -eq 1 ] && return 0
|
|
return 1
|
|
else
|
|
log " BLOCKER: status unreachable. exact command:"
|
|
if [ -n "$STATUS_SSH" ]; then
|
|
log " ssh $STATUS_SSH curl -fsS --connect-timeout 10 $STATUS_URL"
|
|
else
|
|
log " curl -fsS --connect-timeout 10 $STATUS_URL"
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# run_preflight
|
|
# Orchestrates all preflight checks in order: tools -> config -> endpoint -> status.
|
|
# Returns 0 on success, 3 on preflight BLOCKED.
|
|
run_preflight() {
|
|
log "=== PREFLIGHT ==="
|
|
local pre="$OUT_DIR/preflight_${RUN_STAMP}.txt"
|
|
local models_out="$OUT_DIR/models_${RUN_STAMP}.json"
|
|
local status_out="$OUT_DIR/preflight_status_${RUN_STAMP}.json"
|
|
local ok=1
|
|
|
|
# Step 1: tool availability
|
|
if ! preflight_check_tools; then
|
|
ok=0
|
|
fi
|
|
|
|
# Step 2: config check
|
|
local cfg_rc=0
|
|
preflight_check_config "$pre" || cfg_rc=$?
|
|
if [ "$cfg_rc" -ne 0 ]; then
|
|
log " BLOCKER: edge config check failed (rc=$cfg_rc, config=$CONFIG)"
|
|
ok=0
|
|
fi
|
|
|
|
# Step 3: endpoint reachability
|
|
if ! preflight_check_endpoint "$models_out"; then
|
|
ok=0
|
|
fi
|
|
|
|
# Step 4: status reachability + provider health + capacity
|
|
if ! preflight_check_status "$status_out"; then
|
|
ok=0
|
|
fi
|
|
|
|
log "expected baseline: normal_capacity_total=$NORMAL_CAPACITY_TOTAL long_slot_total=$LONG_SLOT_TOTAL"
|
|
if [ "$ok" -eq 1 ]; then
|
|
log "=== PREFLIGHT OK ==="
|
|
return 0
|
|
fi
|
|
log "=== PREFLIGHT BLOCKED (see out-dir; blockers are verification blockers, not user-review) ==="
|
|
return 3
|
|
}
|
|
|
|
# ── scenarios ────────────────────────────────────────────────────────────────
|
|
run_scenario() {
|
|
local name="$1"
|
|
local rc=0
|
|
case "$name" in
|
|
normal-10)
|
|
log "=== SCENARIO normal-10 (expect peak in_flight>=$NORMAL_CAPACITY_TOTAL, queued>=1) ==="
|
|
local peaks="$OUT_DIR/normal-10_peaks_${RUN_STAMP}.txt"
|
|
fire_and_wait normal 10 "normal-10" || rc=1
|
|
assert_peak "$peaks" peak_in_flight -ge "$NORMAL_CAPACITY_TOTAL" "normal-10" || rc=1
|
|
assert_peak "$peaks" peak_queued -ge 1 "normal-10" || rc=1
|
|
assert_recovery "normal-10" || rc=1
|
|
;;
|
|
mixed)
|
|
log "=== SCENARIO mixed (long slots held; normal must not head-of-line block) ==="
|
|
local peaks="$OUT_DIR/mixed_peaks_${RUN_STAMP}.txt"
|
|
# Fill all long slots, and pile normal requests behind them.
|
|
local i pids=()
|
|
for ((i = 1; i <= LONG_SLOT_TOTAL; i++)); do
|
|
fire_request long "$i" "$OUT_DIR/mixed_long_${i}_${RUN_STAMP}" &
|
|
pids+=("$!")
|
|
done
|
|
for ((i = 1; i <= 6; i++)); do
|
|
fire_request normal "$i" "$OUT_DIR/mixed_normal_${i}_${RUN_STAMP}" &
|
|
pids+=("$!")
|
|
done
|
|
poll_peaks "mixed" "$peaks" &
|
|
local poll_pid=$!
|
|
for p in "${pids[@]}"; do wait "$p" 2>/dev/null || true; done
|
|
touch "$OUT_DIR/mixed.polling_done"
|
|
wait "$poll_pid" 2>/dev/null || true
|
|
local lok=0 nok=0
|
|
for ((i = 1; i <= LONG_SLOT_TOTAL; i++)); do
|
|
[ "$(cat "$OUT_DIR/mixed_long_${i}_${RUN_STAMP}.code" 2>/dev/null || echo 000)" = "200" ] && lok=$((lok + 1))
|
|
done
|
|
for ((i = 1; i <= 6; i++)); do
|
|
[ "$(cat "$OUT_DIR/mixed_normal_${i}_${RUN_STAMP}.code" 2>/dev/null || echo 000)" = "200" ] && nok=$((nok + 1))
|
|
done
|
|
log "mixed: long http_200=$lok/$LONG_SLOT_TOTAL normal http_200=$nok/6 (head-of-line blocking would show normal stalling behind full long slots)"
|
|
[ "$lok" -eq "$LONG_SLOT_TOTAL" ] || { log "mixed: FAIL long http_200=$lok/$LONG_SLOT_TOTAL"; rc=1; }
|
|
[ "$nok" -eq 6 ] || { log "mixed: FAIL normal http_200=$nok/6"; rc=1; }
|
|
# long_in_flight is optional in the Control Plane status view: assert
|
|
# only when visible, otherwise defer to Edge dispatch log evidence.
|
|
if [[ "$(read_kv "$peaks" peak_long_in_flight)" =~ ^[0-9]+$ ]]; then
|
|
assert_peak "$peaks" peak_long_in_flight -ge "$LONG_SLOT_TOTAL" "mixed" || rc=1
|
|
else
|
|
log "mixed: long_in_flight not exposed by status view; capture Edge dispatch log (context_class) as review evidence."
|
|
fi
|
|
assert_recovery "mixed" || rc=1
|
|
;;
|
|
all-long-slot-full)
|
|
local n=$((LONG_SLOT_TOTAL + 2))
|
|
log "=== SCENARIO all-long-slot-full (fire $n long > $LONG_SLOT_TOTAL slots; expect queue then dispatch) ==="
|
|
local peaks="$OUT_DIR/all-long-slot-full_peaks_${RUN_STAMP}.txt"
|
|
fire_and_wait long "$n" "all-long-slot-full" || rc=1
|
|
# only long requests are in flight, so total in_flight must not exceed
|
|
# the long-slot total; at least one long request must have queued.
|
|
assert_peak "$peaks" peak_in_flight -le "$LONG_SLOT_TOTAL" "all-long-slot-full" || rc=1
|
|
assert_peak "$peaks" peak_queued -ge 1 "all-long-slot-full" || rc=1
|
|
if [[ "$(read_kv "$peaks" peak_long_queued)" =~ ^[0-9]+$ ]]; then
|
|
assert_peak "$peaks" peak_long_queued -ge 1 "all-long-slot-full" || rc=1
|
|
else
|
|
log "all-long-slot-full: long_queued not exposed by status view; capture Edge dispatch log (queue_reason=long_context_capacity_full) as review evidence."
|
|
fi
|
|
assert_recovery "all-long-slot-full" || rc=1
|
|
;;
|
|
*)
|
|
die "unknown scenario: $name (normal-10 | mixed | all-long-slot-full)"
|
|
;;
|
|
esac
|
|
return "$rc"
|
|
}
|
|
|
|
# ── main ─────────────────────────────────────────────────────────────────────
|
|
rc=0
|
|
if [ "$DO_PREFLIGHT" -eq 1 ]; then
|
|
run_preflight || rc=$?
|
|
fi
|
|
if [ -n "$SCENARIO" ]; then
|
|
if [ "$rc" -ne 0 ]; then
|
|
die "preflight blocked (rc=$rc); not running scenario '$SCENARIO'"
|
|
fi
|
|
run_scenario "$SCENARIO" || rc=$?
|
|
fi
|
|
|
|
log "done rc=$rc evidence=$OUT_DIR"
|
|
exit "$rc"
|