#!/usr/bin/env bash
set -euo pipefail

# OpenAI-compatible Lemonade serving smoke.
#
# Verifies that the Edge OpenAI-compatible input surface converges onto the Node
# `openai_compat` adapter for a Lemonade-style provider: /v1/models lookup,
# non-streaming and streaming /v1/chat/completions, and the `edge smoke openai`
# command path.
#
# Modes (IOP_LEMONADE_MODE):
#   fake (default) - a temporary Go OpenAI-compatible server stands in for
#                    Lemonade so the route can be checked with no external deps.
#   real           - an external Lemonade server is used. Required env:
#                      IOP_LEMONADE_ENDPOINT (root or trailing /v1 both allowed)
#                      IOP_LEMONADE_MODEL
#                    Optional auth (both must be set):
#                      IOP_LEMONADE_AUTH_HEADER_NAME
#                      IOP_LEMONADE_AUTH_HEADER_VALUE
#                    Auth header values are written only to the generated temp
#                    config and never printed to stdout/stderr.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
TMP_ROOT="${IOP_LEMONADE_TMP_ROOT:-$REPO_ROOT/.local}"
mkdir -p "$TMP_ROOT"
TMP_DIR=$(mktemp -d "$TMP_ROOT/e2e-openai-lemonade.XXXXXX")
mkdir -p "$TMP_DIR/go-tmp"
export GOTMPDIR="$TMP_DIR/go-tmp"
export GOCACHE="${IOP_LEMONADE_GOCACHE:-$REPO_ROOT/.gocache}"
mkdir -p "$GOCACHE"

MODE="${IOP_LEMONADE_MODE:-fake}"

EDGE_PID=""
NODE_PID=""
LEMONADE_PID=""
EDGE_FD_OPEN=0

cleanup() {
    if [ "$EDGE_FD_OPEN" -eq 1 ]; then
        { echo "/exit" >&3; } 2>/dev/null || true
        exec 3>&- 2>/dev/null || true
    fi
    if [ -n "$EDGE_PID" ]; then wait "$EDGE_PID" 2>/dev/null || true; fi
    if [ -n "$NODE_PID" ]; then
        kill "$NODE_PID" 2>/dev/null || true
        wait "$NODE_PID" 2>/dev/null || true
    fi
    if [ -n "$LEMONADE_PID" ]; then
        kill "$LEMONADE_PID" 2>/dev/null || true
        wait "$LEMONADE_PID" 2>/dev/null || true
    fi
    if [ "${IOP_LEMONADE_KEEP_TMP:-0}" = "1" ]; then
        echo "[openai-lemonade] keeping temp dir: $TMP_DIR"
        return
    fi
    rm -rf "$TMP_DIR" 2>/dev/null || true
}
trap cleanup EXIT

PORT=$((31000 + RANDOM % 5000))
BOOTSTRAP_PORT=$((21000 + RANDOM % 5000))
OPENAI_PORT=$((36000 + RANDOM % 5000))
LEMONADE_PORT=$((41000 + RANDOM % 5000))
EDGE_METRICS_PORT=$((46000 + RANDOM % 5000))
NODE_METRICS_PORT=$((51000 + RANDOM % 5000))

wait_port() {
    local host="$1"
    local port="$2"
    local label="$3"
    local deadline=$((SECONDS + 20))
    while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/"$1"/"$2"' _ "$host" "$port" 2>/dev/null; do
        if (( SECONDS >= deadline )); then
            echo "[openai-lemonade] $label did not open on $host:$port"
            return 1
        fi
        sleep 0.2
    done
}

# Resolve provider endpoint and model based on mode.
if [ "$MODE" = "real" ]; then
    : "${IOP_LEMONADE_ENDPOINT:?real mode requires IOP_LEMONADE_ENDPOINT}"
    : "${IOP_LEMONADE_MODEL:?real mode requires IOP_LEMONADE_MODEL}"
    NODE_ENDPOINT="$IOP_LEMONADE_ENDPOINT"
    MODEL="$IOP_LEMONADE_MODEL"
    echo "[openai-lemonade] real mode against external Lemonade endpoint (model=$MODEL)"
else
    MODEL="fake-lemonade-model"
    NODE_ENDPOINT="http://127.0.0.1:$LEMONADE_PORT"

    FAKE_LEMONADE="$TMP_DIR/fake_lemonade.go"
    cat > "$FAKE_LEMONADE" <<'EOF'
package main

import (
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"sync/atomic"
)

type chatRequest struct {
Model    string `json:"model"`
Stream   bool   `json:"stream"`
Messages []struct {
Role    string `json:"role"`
Content string `json:"content"`
} `json:"messages"`
}

func main() {
if len(os.Args) < 4 {
log.Fatal("usage: fake_lemonade <listen-addr> <model> <authorization>")
}
model := os.Args[2]
authorization := os.Args[3]
var modelRequests atomic.Int64
var chatRequests atomic.Int64
var responseRequests atomic.Int64
var hangRequests atomic.Int64
var hangCancellations atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.RequestURI != "/v1/models" {
http.Error(w, "unexpected models request", http.StatusBadRequest)
return
}
if r.Header.Get("Authorization") != authorization {
http.Error(w, "unexpected authorization", http.StatusUnauthorized)
return
}
modelRequests.Add(1)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"` + model + `"},{"id":"lemonade-profile-alias"}]}`))
})
mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.RequestURI != "/v1/chat/completions" {
http.Error(w, "unexpected chat request", http.StatusBadRequest)
return
}
if r.Header.Get("Authorization") != authorization {
http.Error(w, "unexpected authorization", http.StatusUnauthorized)
return
}
var req chatRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if req.Model != model {
http.Error(w, "unexpected model: "+req.Model, http.StatusBadRequest)
return
}
chatRequests.Add(1)
if !req.Stream {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"chatcmpl-fake","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","reasoning_content":"thinking...","content":"IOP_OPENAI_LEMONADE_OK"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}`))
return
}
w.Header().Set("Content-Type", "text/event-stream")
flush := func() {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
if len(req.Messages) == 1 && req.Messages[0].Content == "IOP_HANG_AFTER_FINISH" {
hangRequests.Add(1)
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"IOP_HANG_VISIBLE_PREFIX"},"finish_reason":"stop"}]}` + "\n\n"))
flush()
<-r.Context().Done()
hangCancellations.Add(1)
log.Print("IOP_HANG_PROVIDER_RAW_DIAGNOSTIC")
return
}
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"reasoning_content":"thinking..."}}]}` + "\n\n"))
flush()
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"IOP_OPENAI_"}}]}` + "\n\n"))
flush()
_, _ = w.Write([]byte("data: " + `{"choices":[{"delta":{"content":"LEMONADE_OK"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}` + "\n\n"))
flush()
_, _ = w.Write([]byte("data: [DONE]\n\n"))
flush()
})
mux.HandleFunc("/v1/responses", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.RequestURI != "/v1/responses" {
http.Error(w, "unexpected responses request", http.StatusBadRequest)
return
}
if r.Header.Get("Authorization") != authorization {
http.Error(w, "unexpected authorization", http.StatusUnauthorized)
return
}
var req struct { Model string `json:"model"` }
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if req.Model != model {
http.Error(w, "unexpected model: "+req.Model, http.StatusBadRequest)
return
}
responseRequests.Add(1)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"resp-fake","object":"response","model":"` + model + `","output_text":"IOP_OPENAI_LEMONADE_OK","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"IOP_OPENAI_LEMONADE_OK"}]}]}`))
})
mux.HandleFunc("/assert", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "unexpected assertion method", http.StatusMethodNotAllowed)
return
}
models := modelRequests.Load()
chats := chatRequests.Load()
responses := responseRequests.Load()
hangs := hangRequests.Load()
cancellations := hangCancellations.Load()
if models < 1 || chats != 4 || responses != 1 || hangs != 1 || cancellations != 1 {
http.Error(w, fmt.Sprintf("unexpected request counts: models=%d chats=%d responses=%d hangs=%d cancellations=%d", models, chats, responses, hangs, cancellations), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"models":%d,"chats":%d,"responses":%d,"hangs":%d,"cancellations":%d}`, models, chats, responses, hangs, cancellations)
})
log.Fatal(http.ListenAndServe(os.Args[1], mux))
}
EOF

    FAKE_AUTHORIZATION="Bearer fake-lemonade-token"
    FAKE_LEMONADE_BIN="$TMP_DIR/fake-lemonade"
    go build -o "$FAKE_LEMONADE_BIN" "$FAKE_LEMONADE"
    "$FAKE_LEMONADE_BIN" "127.0.0.1:$LEMONADE_PORT" "$MODEL" "$FAKE_AUTHORIZATION" > "$TMP_DIR/fake_lemonade.out" 2>&1 &
    LEMONADE_PID=$!
    wait_port 127.0.0.1 "$LEMONADE_PORT" "fake lemonade"
fi

# Build the provider header block. Auth headers, when provided, are written
# only into the generated temp config (never echoed).
PROVIDER_HEADER_BLOCK="$TMP_DIR/provider_header_block.yaml"
{
    if [ "$MODE" = "fake" ]; then
        echo "          headers:"
        echo "            Authorization: \"$FAKE_AUTHORIZATION\""
    elif [ -n "${IOP_LEMONADE_AUTH_HEADER_NAME:-}" ] && [ -n "${IOP_LEMONADE_AUTH_HEADER_VALUE:-}" ]; then
        echo "          headers:"
        echo "            \"$IOP_LEMONADE_AUTH_HEADER_NAME\": \"$IOP_LEMONADE_AUTH_HEADER_VALUE\""
    fi
} > "$PROVIDER_HEADER_BLOCK"

EDGE_CONFIG="$TMP_DIR/edge.yaml"
NODE_CONFIG="$TMP_DIR/node.yaml"
cat > "$EDGE_CONFIG" <<EOF
server:
  listen: "127.0.0.1:$PORT"
metrics:
  port: $EDGE_METRICS_PORT
bootstrap:
  listen: "127.0.0.1:$BOOTSTRAP_PORT"
  artifact_dir: "$TMP_DIR/artifacts"
openai:
  enabled: true
  listen: "127.0.0.1:$OPENAI_PORT"
  provider_id: "lemonade-smoke-provider"
  session_id: "cline"
  timeout_sec: 30
  strict_output: false
protocol_profiles:
  lemonade_smoke:
    base: openai
    model_mapping:
      lemonade-profile-alias: "$MODEL"
models:
  - id: "client-request-model"
    providers:
      lemonade-smoke-provider: "lemonade-profile-alias"
  - id: "$MODEL"
    providers:
      lemonade-smoke-provider: "lemonade-profile-alias"
console:
  session_id: default
nodes:
  - id: test-node
    alias: test-node
    token: test-token
    adapters:
      openai_compat_instances:
        - name: "lemonade-smoke-backing"
          enabled: true
          provider: lemonade
          endpoint: "$NODE_ENDPOINT"
          capacity: 4
$(cat "$PROVIDER_HEADER_BLOCK")
    providers:
      - id: "lemonade-smoke-provider"
        type: lemonade
        category: api
        adapter: "lemonade-smoke-backing"
        profile: lemonade_smoke
        models:
          - "lemonade-profile-alias"
        health: available
        capacity: 4
        response_stall_timeout_ms: 200
EOF

cat > "$NODE_CONFIG" <<EOF
transport:
  edge_addr: "127.0.0.1:$PORT"
  token: test-token
  reconnect_interval: 1s
metrics:
  port: $NODE_METRICS_PORT
node:
  id: test-node
  alias: test-node
EOF

EDGE_OUT="$TMP_DIR/edge.out"
NODE_OUT="$TMP_DIR/node.out"
mkfifo "$TMP_DIR/edge_fifo"
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
EDGE_PID=$!
exec 3> "$TMP_DIR/edge_fifo"
EDGE_FD_OPEN=1

wait_port 127.0.0.1 "$PORT" "edge node transport"
wait_port 127.0.0.1 "$OPENAI_PORT" "edge openai api"

IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 &
NODE_PID=$!

deadline=$((SECONDS + 30))
while ! grep -q '\[node0-evt\] connected reason="registered"' "$EDGE_OUT"; do
    if (( SECONDS >= deadline )); then
        echo "[openai-lemonade] node registration timed out"
        echo "=== EDGE OUTPUT ==="; cat "$EDGE_OUT"
        echo "=== NODE OUTPUT ==="; cat "$NODE_OUT"
        exit 1
    fi
    sleep 0.2
done

# Edge health.
curl -fsS "http://127.0.0.1:$OPENAI_PORT/healthz" > /dev/null

# /v1/models must surface both the client-facing model and provider target.
MODELS_OUT="$TMP_DIR/models.json"
curl -fsS "http://127.0.0.1:$OPENAI_PORT/v1/models" > "$MODELS_OUT"
grep -q "client-request-model" "$MODELS_OUT"
grep -q "$MODEL" "$MODELS_OUT"

# Non-streaming chat completion.
CHAT_OUT="$TMP_DIR/chat.json"
curl -fsS \
    -H "Content-Type: application/json" \
    -d '{"model":"client-request-model","max_tokens":32,"messages":[{"role":"user","content":"Reply with exactly: smoke token"}]}' \
    "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$CHAT_OUT"
if [ "$MODE" = "fake" ]; then
    grep -q "IOP_OPENAI_LEMONADE_OK" "$CHAT_OUT"
    grep -q "reasoning_content" "$CHAT_OUT"
else
    # real mode: reasoning models can emit a reasoning-only assistant payload
    # when max_tokens is intentionally small for smoke speed.
    grep -Eq '"(content|reasoning_content)":"[^"]' "$CHAT_OUT"
fi

# Streaming chat completion: SSE content chunk(s) and a terminating [DONE].
STREAM_OUT="$TMP_DIR/stream.txt"
curl -fsS -N \
    -H "Content-Type: application/json" \
    -d '{"model":"client-request-model","stream":true,"max_tokens":32,"messages":[{"role":"user","content":"Reply with exactly: stream token"}]}' \
    "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$STREAM_OUT"
if [ "$MODE" = "fake" ]; then
    grep -q '"reasoning_content":"thinking..."' "$STREAM_OUT"
    grep -q '"content":"IOP_OPENAI_"' "$STREAM_OUT"
else
    grep -Eq '"(content|reasoning_content)":"[^"]' "$STREAM_OUT"
fi
grep -q 'data: \[DONE\]' "$STREAM_OUT"

if [ "$MODE" = "fake" ]; then
    # Flush a logical finish and deliberately keep the provider response open.
    # Node's 200ms typed watchdog owns the failure before Edge's 30s request
    # timeout. The committed stream must receive one sanitized terminal only.
    HANG_OUT="$TMP_DIR/hang-after-finish.txt"
    HANG_STARTED=$SECONDS
    curl -fsS -N \
        -H "Content-Type: application/json" \
        -d '{"model":"client-request-model","stream":true,"max_tokens":32,"messages":[{"role":"user","content":"IOP_HANG_AFTER_FINISH"}]}' \
        "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$HANG_OUT"
    HANG_ELAPSED=$((SECONDS - HANG_STARTED))
    if (( HANG_ELAPSED >= 10 )); then
        echo "[openai-lemonade] hang fixture exceeded the bounded IOP liveness window"
        exit 1
    fi
    HANG_PREFIX_COUNT=$(grep -c 'IOP_HANG_VISIBLE_PREFIX' "$HANG_OUT" || true)
    HANG_FINISH_COUNT=$(grep -c '"finish_reason":"stop"' "$HANG_OUT" || true)
    HANG_ERROR_COUNT=$(grep -c '"type":"run_error"' "$HANG_OUT" || true)
    HANG_DONE_COUNT=$(grep -c '^data: \[DONE\]$' "$HANG_OUT" || true)
    if [ "$HANG_PREFIX_COUNT" -ne 1 ] || [ "$HANG_FINISH_COUNT" -ne 1 ] || [ "$HANG_ERROR_COUNT" -ne 1 ] || [ "$HANG_DONE_COUNT" -ne 1 ]; then
        echo "[openai-lemonade] hang fixture terminal counts prefix=$HANG_PREFIX_COUNT finish=$HANG_FINISH_COUNT run_error=$HANG_ERROR_COUNT done=$HANG_DONE_COUNT"
        exit 1
    fi
    if grep -q 'IOP_HANG_PROVIDER_RAW_DIAGNOSTIC' "$HANG_OUT"; then
        echo "[openai-lemonade] raw provider diagnostic leaked to the client"
        exit 1
    fi

    # A normal request after cancellation proves the provider handler and the
    # Node/Edge path remain usable after the fenced attempt.
    POST_CANCEL_OUT="$TMP_DIR/post-cancel-stream.txt"
    curl -fsS -N \
        -H "Content-Type: application/json" \
        -d '{"model":"client-request-model","stream":true,"max_tokens":32,"messages":[{"role":"user","content":"Reply after cancellation"}]}' \
        "http://127.0.0.1:$OPENAI_PORT/v1/chat/completions" > "$POST_CANCEL_OUT"
    grep -q '"content":"IOP_OPENAI_' "$POST_CANCEL_OUT"
    [ "$(grep -c '^data: \[DONE\]$' "$POST_CANCEL_OUT")" -eq 1 ]
fi

# Command-path smoke via the edge CLI.
SMOKE_OUT="$TMP_DIR/iop-edge-smoke-openai.txt"
(cd "$REPO_ROOT" && go run ./apps/edge/cmd/edge smoke openai \
    --model "client-request-model" \
    --base-url "http://127.0.0.1:$OPENAI_PORT" \
    --prompt "Reply with exactly: command smoke token" \
    --timeout 60s) > "$SMOKE_OUT"
grep -q "IOP Edge OpenAI Smoke Test SUCCESS!" "$SMOKE_OUT"

if [ "$MODE" = "fake" ]; then
    FAKE_ASSERT_OUT="$TMP_DIR/fake_assert.json"
    curl -fsS "http://127.0.0.1:$LEMONADE_PORT/assert" > "$FAKE_ASSERT_OUT"
    grep -q '"chats":4' "$FAKE_ASSERT_OUT"
    grep -q '"responses":1' "$FAKE_ASSERT_OUT"
    grep -q '"hangs":1' "$FAKE_ASSERT_OUT"
    grep -q '"cancellations":1' "$FAKE_ASSERT_OUT"
fi

FAILURE_MATCHES="$TMP_DIR/failure-matches.txt"
grep -i -E "node reported error|error run_id=|\[[^]]+-evt\] error|panic:" "$EDGE_OUT" "$NODE_OUT" > "$FAILURE_MATCHES" || true
if [ "$MODE" = "fake" ]; then
    UNEXPECTED_FAILURES="$TMP_DIR/unexpected-failures.txt"
    grep -vi -E "response_stalled|provider response stalled" "$FAILURE_MATCHES" > "$UNEXPECTED_FAILURES" || true
else
    UNEXPECTED_FAILURES="$FAILURE_MATCHES"
fi
if [ -s "$UNEXPECTED_FAILURES" ]; then
    echo "[openai-lemonade] detected failure marker"
    echo "[openai-lemonade] raw runtime output retained only in the temporary artifact directory"
    exit 1
fi

if [ "$MODE" = "fake" ]; then
    grep -qi -E "response_stalled|provider response stalled" "$EDGE_OUT" "$NODE_OUT"
    kill "$LEMONADE_PID"
    wait "$LEMONADE_PID" 2>/dev/null || true
    if kill -0 "$LEMONADE_PID" 2>/dev/null; then
        echo "[openai-lemonade] fake provider survived teardown"
        exit 1
    fi
    LEMONADE_PID=""
fi

echo "[openai-lemonade] OpenAI-compatible Lemonade serving test PASSED (mode=$MODE)."