iop/scripts/e2e-openai-vllm.sh

#!/usr/bin/env bash
set -euo pipefail

# OpenAI-compatible vLLM serving smoke.
#
# Verifies that the Edge OpenAI-compatible input surface converges onto the Node
# `openai_compat` adapter for a vLLM provider: /v1/models lookup, non-streaming
# and streaming /v1/chat/completions using the route alias `qwen3.6:35b` and
# served model `nvidia/Qwen3.6-35B-A3B-NVFP4`.
#
# Modes (IOP_VLLM_MODE):
#   fake (default) - a temporary Go OpenAI-compatible server stands in for
#                    the vLLM provider so the route can be checked with no
#                    external deps.
#   real           - an external vLLM server is used. Required env:
#                      IOP_VLLM_ENDPOINT   (root URL; trailing /v1 both allowed)
#                    Optional:
#                      IOP_VLLM_SERVED_MODEL (default: nvidia/Qwen3.6-35B-A3B-NVFP4)
#                    Auth headers (both must be set together):
#                      IOP_VLLM_AUTH_HEADER_NAME
#                      IOP_VLLM_AUTH_HEADER_VALUE
#                    Auth header values are written only to the generated temp
#                    config and never printed to stdout/stderr.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
TMP_DIR=$(mktemp -d)

MODE="${IOP_VLLM_MODE:-fake}"

ROUTE_ALIAS="qwen3.6:35b"
DEFAULT_SERVED_MODEL="nvidia/Qwen3.6-35B-A3B-NVFP4"

EDGE_PID=""
NODE_PID=""
FAKE_VLLM_PID=""
EDGE_FD_OPEN=0

cleanup() {
    if [ "$EDGE_FD_OPEN" -eq 1 ]; then
        { echo "/exit" >&3; } 2>/dev/null || true
        exec 3>&- 2>/dev/null || true
    fi
    if [ -n "$EDGE_PID" ]; then wait "$EDGE_PID" 2>/dev/null || true; fi
    if [ -n "$NODE_PID" ]; then kill "$NODE_PID" 2>/dev/null || true; fi
    if [ -n "$FAKE_VLLM_PID" ]; then kill "$FAKE_VLLM_PID" 2>/dev/null || true; fi
    if [ "${IOP_VLLM_KEEP_TMP:-0}" = "1" ]; then
        echo "[openai-vllm] keeping temp dir: $TMP_DIR"
        return
    fi
    rm -rf "$TMP_DIR" 2>/dev/null || true
}
trap cleanup EXIT

PORT=$((31000 + RANDOM % 5000))
BOOTSTRAP_PORT=$((21000 + RANDOM % 5000))
OPENAI_PORT=$((36000 + RANDOM % 5000))
VLLM_PORT=$((41000 + RANDOM % 5000))
EDGE_METRICS_PORT=$((46000 + RANDOM % 5000))
NODE_METRICS_PORT=$((51000 + RANDOM % 5000))

wait_port() {
    local host="$1"
    local port="$2"
    local label="$3"
    local deadline=$((SECONDS + 20))
    while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/"$1"/"$2"' _ "$host" "$port" 2>/dev/null; do
        if (( SECONDS >= deadline )); then
            echo "[openai-vllm] $label did not open on $host:$port"
            return 1
        fi
        sleep 0.2
    done
}

# Resolve provider endpoint and served model based on mode.
if [ "$MODE" = "real" ]; then
    : "${IOP_VLLM_ENDPOINT:?real mode requires IOP_VLLM_ENDPOINT}"
    NODE_ENDPOINT="$IOP_VLLM_ENDPOINT"
    SERVED_MODEL="${IOP_VLLM_SERVED_MODEL:-$DEFAULT_SERVED_MODEL}"
    echo "[openai-vllm] real mode against external vLLM endpoint (served_model=$SERVED_MODEL)"
else
    SERVED_MODEL="$DEFAULT_SERVED_MODEL"
    NODE_ENDPOINT="http://127.0.0.1:$VLLM_PORT"

    FAKE_VLLM_SRC="$TMP_DIR/fake_vllm.go"
    cat > "$FAKE_VLLM_SRC" <<'EOF'
package main

import (
"encoding/json"
"log"
"net/http"
"os"
)

type chatRequest struct {
Model    string `json:"model"`
Stream   bool   `json:"stream"`
Messages []struct {
Role    string `json:"role"`
Content string `json:"content"`
} `json:"messages"`
}

func main() {
if len(os.Args) < 3 {
log.Fatal("usage: fake_vllm <listen-addr> <served-model>")
}
servedModel := os.Args[2]
mux := http.NewServeMux()
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"` + servedModel + `"}]}`))
})
mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
var req chatRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if req.Model != servedModel {
http.Error(w, "unexpected model: "+req.Model, http.StatusBadRequest)
return
}
if !req.Stream {
http.Error(w, "expected stream=true from openai_compat adapter", http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "text/event-stream")
flush := func() {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"IOP_OPENAI_"},"finish_reason":null}]}` + "\n\n"))
flush()
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"VLLM_OK"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}` + "\n\n"))
flush()
_, _ = w.Write([]byte("data: [DONE]\n\n"))
flush()
})
log.Fatal(http.ListenAndServe(os.Args[1], mux))
}
EOF

    go run "$FAKE_VLLM_SRC" "127.0.0.1:$VLLM_PORT" "$SERVED_MODEL" > "$TMP_DIR/fake_vllm.out" 2>&1 &
    FAKE_VLLM_PID=$!
    wait_port 127.0.0.1 "$VLLM_PORT" "fake vllm"
fi

# Build the node openai_compat_instances adapter block.
# Auth headers, when provided, are written only into the generated temp config.
ADAPTER_BLOCK="$TMP_DIR/adapter_block.yaml"
{
    echo "      openai_compat_instances:"
    echo "        - name: vllm-local"
    echo "          enabled: true"
    echo "          provider: vllm"
    echo "          endpoint: \"$NODE_ENDPOINT\""
    echo "          capacity: 4"
    echo "          queue_timeout_ms: 5000"
    if [ -n "${IOP_VLLM_AUTH_HEADER_NAME:-}" ] && [ -n "${IOP_VLLM_AUTH_HEADER_VALUE:-}" ]; then
        echo "          headers:"
        echo "            \"$IOP_VLLM_AUTH_HEADER_NAME\": \"$IOP_VLLM_AUTH_HEADER_VALUE\""
    fi
} > "$ADAPTER_BLOCK"

EDGE_CONFIG="$TMP_DIR/edge.yaml"
NODE_CONFIG="$TMP_DIR/node.yaml"
cat > "$EDGE_CONFIG" <<EOF
server:
  listen: "127.0.0.1:$PORT"
metrics:
  port: $EDGE_METRICS_PORT
bootstrap:
  listen: "127.0.0.1:$BOOTSTRAP_PORT"
  artifact_dir: "$TMP_DIR/artifacts"
openai:
  enabled: true
  listen: "127.0.0.1:$OPENAI_PORT"
  adapter: "openai_compat"
  session_id: "smoke"
  timeout_sec: 30
  strict_output: false
  model_routes:
    - model: "$ROUTE_ALIAS"
      adapter: "openai_compat"
      target: "$SERVED_MODEL"
      node: "test-node"
console:
  adapter: openai_compat
  target: "$SERVED_MODEL"
  session_id: default
nodes:
  - id: test-node
    alias: test-node
    token: test-token
    adapters:
$(cat "$ADAPTER_BLOCK")
EOF

cat > "$NODE_CONFIG" <<EOF
transport:
  edge_addr: "127.0.0.1:$PORT"
  token: test-token
  reconnect_interval: 1s
metrics:
  port: $NODE_METRICS_PORT
node:
  id: test-node
  alias: test-node
EOF

EDGE_OUT="$TMP_DIR/edge.out"
NODE_OUT="$TMP_DIR/node.out"
mkfifo "$TMP_DIR/edge_fifo"
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
EDGE_PID=$!
exec 3> "$TMP_DIR/edge_fifo"
EDGE_FD_OPEN=1

wait_port 127.0.0.1 "$PORT" "edge node transport"
wait_port 127.0.0.1 "$OPENAI_PORT" "edge openai api"

IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 &
NODE_PID=$!

deadline=$((SECONDS + 30))
while ! grep -q '\[node0-evt\] connected reason="registered"' "$EDGE_OUT"; do
    if (( SECONDS >= deadline )); then
        echo "[openai-vllm] node registration timed out"
        echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
        echo "=== NODE OUTPUT ==="; cat "$NODE_OUT"
        exit 1
    fi
    sleep 0.2
done

# Edge health.
curl -fsS "http://127.0.0.1:$OPENAI_PORT/healthz" > /dev/null

# /v1/models must surface the route alias.
MODELS_OUT="$TMP_DIR/models.json"
curl -fsS "http://127.0.0.1:$OPENAI_PORT/v1/models" > "$MODELS_OUT"
grep -q "$ROUTE_ALIAS" "$MODELS_OUT"

# Non-streaming chat completion using the route alias.
CHAT_OUT="$TMP_DIR/chat.json"
curl -fsS \
    -H "Content-Type: application/json" \
    -d "{\"model\":\"$ROUTE_ALIAS\",\"max_tokens\":32,\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: smoke token\"}]}" \
    "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$CHAT_OUT"
if [ "$MODE" = "fake" ]; then
    grep -q "IOP_OPENAI_VLLM_OK" "$CHAT_OUT"
else
    grep -Eq '"content":"[^"]' "$CHAT_OUT"
fi
grep -q '"finish_reason":"' "$CHAT_OUT"

# Streaming chat completion: SSE chunks, finish_reason, and terminating [DONE].
STREAM_OUT="$TMP_DIR/stream.txt"
curl -fsS -N \
    -H "Content-Type: application/json" \
    -d "{\"model\":\"$ROUTE_ALIAS\",\"stream\":true,\"max_tokens\":32,\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: stream token\"}]}" \
    "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$STREAM_OUT"
if [ "$MODE" = "fake" ]; then
    grep -q '"content":"IOP_OPENAI_"' "$STREAM_OUT"
fi
grep -q '"finish_reason":"' "$STREAM_OUT"
grep -q 'data: \[DONE\]' "$STREAM_OUT"

if grep -i -E "node reported error|error run_id=|\[[^]]+-evt\] error|panic:" "$EDGE_OUT" "$NODE_OUT" >/dev/null; then
    echo "[openai-vllm] detected failure marker"
    echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
    echo "=== NODE OUTPUT ==="; cat "$NODE_OUT"
    exit 1
fi

echo "[openai-vllm] OpenAI-compatible vLLM serving test PASSED (mode=$MODE)."