toki
f98d1b434b
Stream Gate 준비 실패에서도 요청 terminal 관측 계약을 지키고, direct provider identity 마이그레이션 누락으로 기존 검증과 운영 설정이 깨지지 않게 한다.
315 lines
10 KiB
Bash
Executable file
315 lines
10 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# OpenAI-compatible vLLM serving smoke.
|
|
#
|
|
# Verifies that the Edge OpenAI-compatible input surface converges onto the Node
|
|
# `openai_compat` adapter for a vLLM provider: /v1/models lookup, non-streaming
|
|
# and streaming /v1/chat/completions using the route alias `qwen3.6:35b` and
|
|
# served model `nvidia/Qwen3.6-35B-A3B-NVFP4`.
|
|
#
|
|
# Modes (IOP_VLLM_MODE):
|
|
# fake (default) - a temporary Go OpenAI-compatible server stands in for
|
|
# the vLLM provider so the route can be checked with no
|
|
# external deps.
|
|
# real - an external vLLM server is used. Required env:
|
|
# IOP_VLLM_ENDPOINT (root URL; trailing /v1 both allowed)
|
|
# Optional:
|
|
# IOP_VLLM_SERVED_MODEL (default: nvidia/Qwen3.6-35B-A3B-NVFP4)
|
|
# Auth headers (both must be set together):
|
|
# IOP_VLLM_AUTH_HEADER_NAME
|
|
# IOP_VLLM_AUTH_HEADER_VALUE
|
|
# Auth header values are written only to the generated temp
|
|
# config and never printed to stdout/stderr.
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
TMP_DIR=$(mktemp -d)
|
|
|
|
MODE="${IOP_VLLM_MODE:-fake}"
|
|
|
|
ROUTE_ALIAS="qwen3.6:35b"
|
|
DEFAULT_SERVED_MODEL="nvidia/Qwen3.6-35B-A3B-NVFP4"
|
|
RAW_LOG_SENTINEL="IOP_RAW_SENTINEL_DO_NOT_LOG"
|
|
|
|
EDGE_PID=""
|
|
NODE_PID=""
|
|
FAKE_VLLM_PID=""
|
|
EDGE_FD_OPEN=0
|
|
|
|
cleanup() {
|
|
if [ "$EDGE_FD_OPEN" -eq 1 ]; then
|
|
{ echo "/exit" >&3; } 2>/dev/null || true
|
|
exec 3>&- 2>/dev/null || true
|
|
fi
|
|
if [ -n "$EDGE_PID" ]; then wait "$EDGE_PID" 2>/dev/null || true; fi
|
|
if [ -n "$NODE_PID" ]; then kill "$NODE_PID" 2>/dev/null || true; fi
|
|
if [ -n "$FAKE_VLLM_PID" ]; then kill "$FAKE_VLLM_PID" 2>/dev/null || true; fi
|
|
if [ "${IOP_VLLM_KEEP_TMP:-0}" = "1" ]; then
|
|
echo "[openai-vllm] keeping temp dir: $TMP_DIR"
|
|
return
|
|
fi
|
|
rm -rf "$TMP_DIR" 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
PORT=$((31000 + RANDOM % 5000))
|
|
BOOTSTRAP_PORT=$((21000 + RANDOM % 5000))
|
|
OPENAI_PORT=$((36000 + RANDOM % 5000))
|
|
VLLM_PORT=$((41000 + RANDOM % 5000))
|
|
EDGE_METRICS_PORT=$((46000 + RANDOM % 5000))
|
|
NODE_METRICS_PORT=$((51000 + RANDOM % 5000))
|
|
|
|
wait_port() {
|
|
local host="$1"
|
|
local port="$2"
|
|
local label="$3"
|
|
local deadline=$((SECONDS + 20))
|
|
while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/"$1"/"$2"' _ "$host" "$port" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[openai-vllm] $label did not open on $host:$port"
|
|
return 1
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
}
|
|
|
|
# Resolve provider endpoint and served model based on mode.
|
|
if [ "$MODE" = "real" ]; then
|
|
: "${IOP_VLLM_ENDPOINT:?real mode requires IOP_VLLM_ENDPOINT}"
|
|
NODE_ENDPOINT="$IOP_VLLM_ENDPOINT"
|
|
SERVED_MODEL="${IOP_VLLM_SERVED_MODEL:-$DEFAULT_SERVED_MODEL}"
|
|
echo "[openai-vllm] real mode against external vLLM endpoint (served_model=$SERVED_MODEL)"
|
|
else
|
|
SERVED_MODEL="$DEFAULT_SERVED_MODEL"
|
|
NODE_ENDPOINT="http://127.0.0.1:$VLLM_PORT"
|
|
|
|
FAKE_VLLM_SRC="$TMP_DIR/fake_vllm.go"
|
|
cat > "$FAKE_VLLM_SRC" <<'EOF'
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
)
|
|
|
|
type chatRequest struct {
|
|
Model string `json:"model"`
|
|
Stream bool `json:"stream"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
|
|
func main() {
|
|
if len(os.Args) < 3 {
|
|
log.Fatal("usage: fake_vllm <listen-addr> <served-model>")
|
|
}
|
|
servedModel := os.Args[2]
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"` + servedModel + `"}]}`))
|
|
})
|
|
mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
|
var req chatRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if req.Model != servedModel {
|
|
http.Error(w, "unexpected model: "+req.Model, http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !req.Stream {
|
|
// The openai_compat route relays a non-streaming request verbatim, so the
|
|
// provider must answer it with a standard chat.completion JSON object. The
|
|
// content matches the streaming fixture so both assertions look for the
|
|
// same marker.
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"chatcmpl-fake","object":"chat.completion","created":0,"model":"` + servedModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"IOP_OPENAI_VLLM_OK"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
flush := func() {
|
|
if f, ok := w.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
}
|
|
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"IOP_OPENAI_"},"finish_reason":null}]}` + "\n\n"))
|
|
flush()
|
|
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"VLLM_OK"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}` + "\n\n"))
|
|
flush()
|
|
_, _ = w.Write([]byte("data: [DONE]\n\n"))
|
|
flush()
|
|
})
|
|
log.Fatal(http.ListenAndServe(os.Args[1], mux))
|
|
}
|
|
EOF
|
|
|
|
go run "$FAKE_VLLM_SRC" "127.0.0.1:$VLLM_PORT" "$SERVED_MODEL" > "$TMP_DIR/fake_vllm.out" 2>&1 &
|
|
FAKE_VLLM_PID=$!
|
|
wait_port 127.0.0.1 "$VLLM_PORT" "fake vllm"
|
|
fi
|
|
|
|
# Build the node openai_compat_instances adapter block.
|
|
# Auth headers, when provided, are written only into the generated temp config.
|
|
ADAPTER_BLOCK="$TMP_DIR/adapter_block.yaml"
|
|
{
|
|
echo " openai_compat_instances:"
|
|
echo " - name: vllm-local"
|
|
echo " enabled: true"
|
|
echo " provider: vllm"
|
|
echo " endpoint: \"$NODE_ENDPOINT\""
|
|
echo " capacity: 4"
|
|
echo " queue_timeout_ms: 5000"
|
|
if [ -n "${IOP_VLLM_AUTH_HEADER_NAME:-}" ] && [ -n "${IOP_VLLM_AUTH_HEADER_VALUE:-}" ]; then
|
|
echo " headers:"
|
|
echo " \"$IOP_VLLM_AUTH_HEADER_NAME\": \"$IOP_VLLM_AUTH_HEADER_VALUE\""
|
|
fi
|
|
} > "$ADAPTER_BLOCK"
|
|
|
|
EDGE_CONFIG="$TMP_DIR/edge.yaml"
|
|
EDGE_LOG="$TMP_DIR/edge.log"
|
|
NODE_CONFIG="$TMP_DIR/node.yaml"
|
|
cat > "$EDGE_CONFIG" <<EOF
|
|
server:
|
|
listen: "127.0.0.1:$PORT"
|
|
logging:
|
|
level: info
|
|
pretty: false
|
|
path: "$EDGE_LOG"
|
|
metrics:
|
|
port: $EDGE_METRICS_PORT
|
|
bootstrap:
|
|
listen: "127.0.0.1:$BOOTSTRAP_PORT"
|
|
artifact_dir: "$TMP_DIR/artifacts"
|
|
openai:
|
|
enabled: true
|
|
listen: "127.0.0.1:$OPENAI_PORT"
|
|
provider_id: "vllm-smoke-provider"
|
|
adapter: "openai_compat"
|
|
session_id: "smoke"
|
|
timeout_sec: 30
|
|
strict_output: false
|
|
stream_evidence_gate:
|
|
enabled: true
|
|
max_request_fault_recovery: 3
|
|
model_routes:
|
|
- model: "$ROUTE_ALIAS"
|
|
adapter: "openai_compat"
|
|
target: "$SERVED_MODEL"
|
|
node: "test-node"
|
|
console:
|
|
adapter: openai_compat
|
|
target: "$SERVED_MODEL"
|
|
session_id: default
|
|
nodes:
|
|
- id: test-node
|
|
alias: test-node
|
|
token: test-token
|
|
adapters:
|
|
$(cat "$ADAPTER_BLOCK")
|
|
EOF
|
|
|
|
cat > "$NODE_CONFIG" <<EOF
|
|
transport:
|
|
edge_addr: "127.0.0.1:$PORT"
|
|
token: test-token
|
|
reconnect_interval: 1s
|
|
metrics:
|
|
port: $NODE_METRICS_PORT
|
|
node:
|
|
id: test-node
|
|
alias: test-node
|
|
EOF
|
|
|
|
EDGE_OUT="$TMP_DIR/edge.out"
|
|
NODE_OUT="$TMP_DIR/node.out"
|
|
mkfifo "$TMP_DIR/edge_fifo"
|
|
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
|
|
EDGE_PID=$!
|
|
exec 3> "$TMP_DIR/edge_fifo"
|
|
EDGE_FD_OPEN=1
|
|
|
|
wait_port 127.0.0.1 "$PORT" "edge node transport"
|
|
wait_port 127.0.0.1 "$OPENAI_PORT" "edge openai api"
|
|
|
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 &
|
|
NODE_PID=$!
|
|
|
|
deadline=$((SECONDS + 30))
|
|
while ! grep -q '\[node0-evt\] connected reason="registered"' "$EDGE_OUT"; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[openai-vllm] node registration timed out"
|
|
echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
|
|
echo "=== NODE OUTPUT ==="; cat "$NODE_OUT"
|
|
exit 1
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
|
|
# Edge health.
|
|
curl -fsS "http://127.0.0.1:$OPENAI_PORT/healthz" > /dev/null
|
|
|
|
# /v1/models must surface the route alias.
|
|
MODELS_OUT="$TMP_DIR/models.json"
|
|
curl -fsS "http://127.0.0.1:$OPENAI_PORT/v1/models" > "$MODELS_OUT"
|
|
grep -q "$ROUTE_ALIAS" "$MODELS_OUT"
|
|
|
|
# Non-streaming chat completion using the route alias.
|
|
CHAT_OUT="$TMP_DIR/chat.json"
|
|
curl -fsS \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"model\":\"$ROUTE_ALIAS\",\"max_tokens\":32,\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: smoke token\"}]}" \
|
|
"http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$CHAT_OUT"
|
|
if [ "$MODE" = "fake" ]; then
|
|
grep -q "IOP_OPENAI_VLLM_OK" "$CHAT_OUT"
|
|
else
|
|
grep -Eq '"content":"[^"]' "$CHAT_OUT"
|
|
fi
|
|
grep -q '"finish_reason":"' "$CHAT_OUT"
|
|
|
|
# Streaming chat completion: SSE chunks, finish_reason, and terminating [DONE].
|
|
STREAM_OUT="$TMP_DIR/stream.txt"
|
|
curl -fsS -N \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"model\":\"$ROUTE_ALIAS\",\"stream\":true,\"max_tokens\":32,\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: $RAW_LOG_SENTINEL\"}]}" \
|
|
"http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$STREAM_OUT"
|
|
if [ "$MODE" = "fake" ]; then
|
|
grep -q '"content":"IOP_OPENAI_"' "$STREAM_OUT"
|
|
fi
|
|
grep -q '"finish_reason":"' "$STREAM_OUT"
|
|
grep -q 'data: \[DONE\]' "$STREAM_OUT"
|
|
|
|
# The fake full-cycle must exercise the production observation sink with
|
|
# request/attempt/actual-target correlation while keeping raw input absent.
|
|
if ! grep -q 'streamgate_filter_observation' "$EDGE_LOG"; then
|
|
echo "[openai-vllm] streamgate observation log missing"
|
|
echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
|
|
echo "=== EDGE LOG ==="; cat "$EDGE_LOG"
|
|
exit 1
|
|
fi
|
|
for field in correlation_id attempt_id model_group actual_model actual_provider execution_path; do
|
|
if ! grep -Eq "\"?$field\"?[:=]" "$EDGE_LOG"; then
|
|
echo "[openai-vllm] observation field missing: $field"
|
|
echo "=== EDGE LOG ==="; cat "$EDGE_LOG"
|
|
exit 1
|
|
fi
|
|
done
|
|
if grep -F "$RAW_LOG_SENTINEL" "$EDGE_OUT" "$EDGE_LOG" "$NODE_OUT" >/dev/null; then
|
|
echo "[openai-vllm] raw request sentinel leaked into runtime logs"
|
|
exit 1
|
|
fi
|
|
|
|
if grep -i -E "node reported error|error run_id=|\[[^]]+-evt\] error|panic:" "$EDGE_OUT" "$EDGE_LOG" "$NODE_OUT" >/dev/null; then
|
|
echo "[openai-vllm] detected failure marker"
|
|
echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
|
|
echo "=== NODE OUTPUT ==="; cat "$NODE_OUT"
|
|
exit 1
|
|
fi
|
|
|
|
echo "[openai-vllm] OpenAI-compatible vLLM serving test PASSED (mode=$MODE)."
|