#!/usr/bin/env bash set -euo pipefail # Long-context admission dev capacity smoke. # # Milestone: model-group-long-context-admission, SDD S09 (capacity-smoke). # Verifies, against the live dev provider pool (GX10 vLLM, OneXPlayer Lemonade, # mac-mlx-vllm), that: # normal-10 - normal capacity+1 concurrency reaches total in_flight and # queues at least one request, then recovers to zero. # mixed - long requests hold long slots while normal requests still # dispatch without head-of-line blocking, then recover. # all-long-slot-full - more long requests than total long slots queue, then # dispatch as long slots free up, then recover to zero. # # This is a live-dependency auxiliary smoke. It does not stand up any process; it # drives an already-running dev Edge OpenAI-compatible endpoint and reads the # Control Plane status snapshot. Every reachability failure is recorded as an # exact failed command (a verification blocker), never turned into a user-review # request. # # Secrets: the bearer token is read from IOP_LONG_SMOKE_TOKEN only and is never # printed to stdout/stderr or written to the out-dir. # # Options: # --preflight run source/config/endpoint/identity/snapshot checks only # --scenario normal-10 | mixed | all-long-slot-full # --config edge config for `edge config check` (default configs/edge.yaml) # --out-dir evidence output dir (default /tmp/iop-long-admission-smoke) # -h | --help usage # # Environment (defaults target the dev-runtime inventory; override per host): # IOP_LONG_SMOKE_BASE_URL Edge OpenAI-compatible base URL (default http://toki-labs.com:18083/v1) # IOP_LONG_SMOKE_STATUS_URL Control Plane status URL (default runner-local edge status) # IOP_LONG_SMOKE_STATUS_SSH optional ssh target used to curl STATUS_URL from a host # that cannot reach it directly (e.g. toki@toki-labs.com) # IOP_LONG_SMOKE_TOKEN optional bearer token for the OpenAI-compatible endpoint # IOP_LONG_SMOKE_MODEL model alias (default qwen3.6:35b) # IOP_LONG_SMOKE_LONG_CHARS synthetic long-prompt size in chars (default 720000 ~ 200k tokens) # IOP_LONG_SMOKE_MAX_TOKENS completion max_tokens per request (default 64) # IOP_LONG_SMOKE_POLL_SECS hard ceiling seconds for the status poll loop; the # loop otherwise runs until requests finish (default 3600) # IOP_LONG_SMOKE_EDGE_BIN optional prebuilt edge binary used for `config check` # instead of `go run` (for runners without a go toolchain) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # REPO_ROOT is where `edge config check` runs. Defaults to the script's parent # repo, but can be overridden (e.g. when the script is copied to /tmp on a runner # whose checkout lives elsewhere). REPO_ROOT="${IOP_LONG_SMOKE_REPO:-$(cd "$SCRIPT_DIR/.." && pwd)}" BASE_URL="${IOP_LONG_SMOKE_BASE_URL:-http://toki-labs.com:18083/v1}" STATUS_URL="${IOP_LONG_SMOKE_STATUS_URL:-http://127.0.0.1:18001/edges/edge-toki-labs-dev/status}" STATUS_SSH="${IOP_LONG_SMOKE_STATUS_SSH:-}" MODEL="${IOP_LONG_SMOKE_MODEL:-qwen3.6:35b}" LONG_CHARS="${IOP_LONG_SMOKE_LONG_CHARS:-720000}" MAX_TOKENS="${IOP_LONG_SMOKE_MAX_TOKENS:-64}" POLL_SECS="${IOP_LONG_SMOKE_POLL_SECS:-3600}" EDGE_BIN="${IOP_LONG_SMOKE_EDGE_BIN:-}" CONFIG="configs/edge.yaml" OUT_DIR="/tmp/iop-long-admission-smoke" SCENARIO="" DO_PREFLIGHT=0 # dev pool baseline (agent-test/inventory-dev.yaml). PROVIDER_IDS=(gx10-vllm onexplayer-lemonade mac-mlx-vllm) NORMAL_CAPACITY_TOTAL=9 # gx10=4 + onexplayer=3 + mac=2 LONG_SLOT_TOTAL=4 # gx10=1 + onexplayer=2 + mac=1 usage() { sed -n '3,54p' "$0" } log() { echo "[long-admission-smoke] $*"; } die() { log "ERROR: $*" exit 1 } # ── argument parsing ───────────────────────────────────────────────────────── while [ $# -gt 0 ]; do case "$1" in --preflight) DO_PREFLIGHT=1; shift ;; --scenario) SCENARIO="${2:?--scenario requires a value}"; shift 2 ;; --config) CONFIG="${2:?--config requires a value}"; shift 2 ;; --out-dir) OUT_DIR="${2:?--out-dir requires a value}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown argument: $1 (see --help)" ;; esac done if [ "$DO_PREFLIGHT" -eq 0 ] && [ -z "$SCENARIO" ]; then die "nothing to do: pass --preflight and/or --scenario " fi command -v curl >/dev/null 2>&1 || die "curl is required" command -v jq >/dev/null 2>&1 || die "jq is required" mkdir -p "$OUT_DIR" RUN_STAMP="$(date -u +%Y%m%dT%H%M%SZ)" log "out-dir=$OUT_DIR run=$RUN_STAMP base_url=$BASE_URL" # ── helpers ────────────────────────────────────────────────────────────────── # curl auth args, built once. Expanded with the ${arr[@]+...} idiom so an empty # array survives `set -u` on the dev runner's macOS bash 3.2, which also lacks # `mapfile` (do not reintroduce bash-4-only builtins here). AUTH_ARGS=() if [ -n "${IOP_LONG_SMOKE_TOKEN:-}" ]; then AUTH_ARGS=(-H "Authorization: Bearer ${IOP_LONG_SMOKE_TOKEN}") fi # fetch_status : capture the Control Plane status JSON. fetch_status() { local out="$1" if [ -n "$STATUS_SSH" ]; then ssh -o BatchMode=yes -o ConnectTimeout=10 "$STATUS_SSH" \ "curl -fsS --connect-timeout 10 '$STATUS_URL'" >"$out" 2>"$out.err" else curl -fsS --connect-timeout 10 "$STATUS_URL" >"$out" 2>"$out.err" fi } # provider_snapshots_jq: normalize provider snapshots from either the edge status # response shape or a fleet node list into a flat array. SNAP_JQ='[.. | objects | select(has("provider_snapshots")) | .provider_snapshots[]?]' # summarize_snapshot -> prints a compact per-provider line and totals. summarize_snapshot() { local file="$1" jq -r " ${SNAP_JQ} as \$s | (\$s | map(.in_flight // 0) | add // 0) as \$inflight | (\$s | map(.queued // 0) | add // 0) as \$queued | (\$s | map(.long_in_flight // 0) | add // 0) as \$loninflight | (\$s | map(.long_queued // 0) | add // 0) as \$lonqueued | (\$s | map(has(\"long_in_flight\")) | any) as \$haslong | (\$s[] | \" provider=\(.id // .adapter // \"?\") capacity=\(.capacity // 0) in_flight=\(.in_flight // 0) queued=\(.queued // 0) long_in_flight=\(.long_in_flight // \"n/a\") long_queued=\(.long_queued // \"n/a\") health=\(.health // \"?\")\"), \" TOTAL in_flight=\(\$inflight) queued=\(\$queued) long_in_flight=\(if \$haslong then \$loninflight else \"n/a\" end) long_queued=\(if \$haslong then \$lonqueued else \"n/a\" end)\" " "$file" 2>/dev/null || echo " (unparseable status json)" } # totals_from_snapshot -> integer sum, or empty when key absent everywhere. totals_from_snapshot() { local file="$1" key="$2" jq -r "${SNAP_JQ} as \$s | if (\$s | map(has(\"${key}\")) | any) then (\$s | map(.${key} // 0) | add // 0) else \"\" end" \ "$file" 2>/dev/null } # build the synthetic long prompt file once (kept in out-dir, not repo). LONG_PROMPT_FILE="$OUT_DIR/long_prompt_${RUN_STAMP}.txt" ensure_long_prompt() { [ -f "$LONG_PROMPT_FILE" ] && return 0 # Repeat a filler sentence to LONG_CHARS. Synthetic text is enough; a # tokenizer-perfect fixture is intentionally out of scope. local unit="Long context admission smoke synthetic filler segment. " local buf="" while [ "${#buf}" -lt "$LONG_CHARS" ]; do buf="${buf}${unit}" done printf '%s' "${buf:0:$LONG_CHARS}" >"$LONG_PROMPT_FILE" log "synthetic long prompt: ${LONG_CHARS} chars -> $LONG_PROMPT_FILE" } # build a chat completion request body file for a given kind (normal|long). # $1=kind $2=idx $3=out-body-file build_body() { local kind="$1" idx="$2" out="$3" content if [ "$kind" = "long" ]; then ensure_long_prompt content="$(cat "$LONG_PROMPT_FILE") Summarize the above in one short sentence. request_idx=${idx}" else content="Reply with the single word OK. request_idx=${idx}" fi jq -n --arg model "$MODEL" --arg content "$content" --argjson maxt "$MAX_TOKENS" \ '{model:$model, max_tokens:$maxt, stream:false, messages:[{role:"user", content:$content}]}' \ >"$out" } # fire one request in the background; write HTTP code + body to out prefix. # $1=kind $2=idx $3=out-prefix fire_request() { local kind="$1" idx="$2" prefix="$3" local body="$prefix.req.json" build_body "$kind" "$idx" "$body" curl -sS -o "$prefix.resp.json" -w '%{http_code}' \ --connect-timeout 15 --max-time 1800 \ -H "Content-Type: application/json" \ ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \ -X POST "$BASE_URL/chat/completions" \ --data-binary "@$body" >"$prefix.code" 2>"$prefix.curl.err" || true } # poll status repeatedly into a peaks file while a scenario runs. The poll loop # runs until the caller creates "$OUT_DIR/