toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/scripts/e2e-smoke.sh
toki a52ecb8efe 기능: OpenAI 서빙과 필드 배포 구성을 추가한다 
필드 테스트를 위해 edge OpenAI-compatible 경로와 node adapter 설정을 확장하고, Jenkins 바이너리 빌드 및 control-plane/web compose 배포 구성을 함께 정리한다.
2026-05-19 16:04:23 +09:00

531 lines
18 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
echo "[e2e] NOTE: auxiliary smoke only; completion requires bin/edge.sh + bin/node.sh user-flow verification."
if command -v shellcheck >/dev/null 2>&1; then
echo "[e2e] running shellcheck..."
shellcheck "$0" "$REPO_ROOT/bin/edge.sh" "$REPO_ROOT/bin/node.sh"
else
echo "[e2e] shellcheck not found, skipping"
fi
TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR" 2>/dev/null || true' EXIT
EDGE_CONFIG="$TMP_DIR/edge.yaml"
NODE_CONFIG="$TMP_DIR/node.yaml"
PORT=$((30000 + RANDOM % 10000))
EDGE_METRICS_PORT=$((40000 + RANDOM % 10000))
NODE_METRICS_PORT=$((50000 + RANDOM % 10000))
PROFILE="${IOP_E2E_PROFILE:-mock}"
IDLE_SECONDS="${IOP_E2E_IDLE_SECONDS:-0}"
REGISTER_TIMEOUT="${IOP_E2E_REGISTER_TIMEOUT:-60}"
EVENT_TIMEOUT="${IOP_E2E_EVENT_TIMEOUT:-30}"
RUN_TIMEOUT="${IOP_E2E_RUN_TIMEOUT:-120}"
STATUS_TIMEOUT="${IOP_E2E_STATUS_TIMEOUT:-$RUN_TIMEOUT}"
COMMAND_SETTLE_SECONDS="${IOP_E2E_COMMAND_SETTLE_SECONDS:-0.2}"
IS_PERSISTENT=0
HAS_STATUS=0
EXPECT_TAIL_MESSAGES=0
FIRST_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_alpha"
FIRST_EXPECTED="SMOKE_ALPHA"
FIRST_TAIL_EXPECTED="IOP_SMOKE_ALPHA_TAIL"
SECOND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_beta"
SECOND_EXPECTED="SMOKE_BETA"
SECOND_TAIL_EXPECTED="IOP_SMOKE_BETA_TAIL"
BACKGROUND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_bg"
BACKGROUND_EXPECTED="SMOKE_BG"
BACKGROUND_TAIL_EXPECTED="IOP_SMOKE_BG_TAIL"
if [ "$PROFILE" = "mock" ]; then
echo "[e2e] preparing honest mock smoke test (using scripted cli adapter)..."
IS_PERSISTENT=1
EXPECT_TAIL_MESSAGES=1
TARGET="fake-cli"
MOCK_CLI="$TMP_DIR/fake-cli.sh"
cat <<'EOF' > "$MOCK_CLI"
#!/usr/bin/env sh
while IFS= read -r line; do
case "$line" in
*iop_smoke_alpha*) printf 'IOP_SMOKE_ALPHA\nIOP_SMOKE_ALPHA_TAIL\n' ;;
*iop_smoke_beta*) printf 'IOP_SMOKE_BETA\nIOP_SMOKE_BETA_TAIL\n' ;;
*iop_smoke_bg*) printf 'IOP_SMOKE_BG\nIOP_SMOKE_BG_TAIL\n' ;;
*) printf 'IOP_SMOKE_UNKNOWN\n' ;;
esac
done
EOF
chmod +x "$MOCK_CLI"
cat <<EOF > "$EDGE_CONFIG"
server:
listen: "127.0.0.1:$PORT"
metrics:
port: $EDGE_METRICS_PORT
nodes:
- id: test-node
alias: test-node
token: test-token
runtime:
workspace_root: "$TMP_DIR/workspace"
adapters:
mock:
enabled: true
cli:
enabled: true
profiles:
fake-cli:
command: "$MOCK_CLI"
persistent: true
response_idle_timeout_ms: 1000
console:
adapter: cli
target: fake-cli
session_id: default
EOF
else
echo "[e2e] preparing real profile smoke test (profile: $PROFILE)..."
TARGET="$PROFILE"
PROFILE_BLOCK=$(awk -v profile="$PROFILE" '
$0 ~ "^[[:space:]]+"profile":" {
found=1;
print $0;
indent=index($0, profile) - 1;
next;
}
found {
if ($0 ~ "^[[:space:]]*$") { print $0; next; }
curr_indent=0;
while(curr_indent < length($0) && substr($0, curr_indent+1, 1) == " ") curr_indent++;
if(curr_indent > indent) {
print $0;
} else {
exit;
}
}
' "$REPO_ROOT/configs/edge.yaml")
if [ -z "$PROFILE_BLOCK" ]; then
echo "[e2e] BLOCKED: Profile '$PROFILE' not found in configs/edge.yaml"
exit 1
fi
if echo "$PROFILE_BLOCK" | grep -q "persistent:[[:space:]]*true"; then
IS_PERSISTENT=1
fi
# Simple check for status support based on command name
if echo "$PROFILE_BLOCK" | grep -i -E "gemini|claude|codex" >/dev/null; then
HAS_STATUS=1
fi
cat <<EOF > "$EDGE_CONFIG"
server:
listen: "127.0.0.1:$PORT"
metrics:
port: $EDGE_METRICS_PORT
nodes:
- id: test-node
alias: test-node
token: test-token
runtime:
workspace_root: "$TMP_DIR/workspace"
adapters:
cli:
enabled: true
profiles:
$PROFILE_BLOCK
console:
adapter: cli
target: $PROFILE
session_id: default
EOF
fi
cat <<EOF > "$NODE_CONFIG"
transport:
edge_addr: "127.0.0.1:$PORT"
token: test-token
reconnect_interval: 1s
metrics:
port: $NODE_METRICS_PORT
node:
id: test-node
alias: test-node
EOF
echo "[e2e] starting smoke test (profile: $PROFILE, port: $PORT, persistent: $IS_PERSISTENT, has_status: $HAS_STATUS)"
EDGE_OUT="$TMP_DIR/edge_out"
NODE_OUT="$TMP_DIR/node_out"
rm -f "$TMP_DIR/edge_fifo"
mkfifo "$TMP_DIR/edge_fifo"
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/bin/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
EDGE_PID=$!
exec 3> "$TMP_DIR/edge_fifo"
deadline=$((SECONDS + 10))
while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/'"$PORT" 2>/dev/null; do
if (( SECONDS >= deadline )); then
echo "[e2e] edge failed to bind port $PORT"
cat "$EDGE_OUT"
exit 1
fi
sleep 0.5
done
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/bin/node.sh" > "$NODE_OUT" 2>&1 &
NODE_PID=$!
LAST_CMD_START_LINE=1
ensure_smoke_processes_running() {
local context="$1"
if ! kill -0 "$EDGE_PID" 2>/dev/null; then
echo "[e2e] edge exited while waiting for $context"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
exit 1
fi
if ! kill -0 "$NODE_PID" 2>/dev/null; then
echo "[e2e] node exited while waiting for $context"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
}
wait_for_node_registration() {
local deadline
deadline=$((SECONDS + REGISTER_TIMEOUT))
echo "[e2e] waiting for node registration (timeout: ${REGISTER_TIMEOUT}s)"
while ! grep -q '\[node0-evt\] connected reason="registered"' "$EDGE_OUT"; do
ensure_smoke_processes_running "node registration"
if (( SECONDS >= deadline )); then
echo "[e2e] node registration timed out"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
wait_for_edge_pattern_since() {
local pattern="$1"
local start_line="$2"
local msg="$3"
local timeout="${4:-$EVENT_TIMEOUT}"
local deadline
local slice
deadline=$((SECONDS + timeout))
slice="$TMP_DIR/edge_wait_slice"
while true; do
tail -n +"$start_line" "$EDGE_OUT" > "$slice" || true
if grep -qE "$pattern" "$slice"; then
return 0
fi
ensure_smoke_processes_running "$msg"
if (( SECONDS >= deadline )); then
echo "[e2e] FAIL: timed out waiting for $msg (pattern: '$pattern', timeout: ${timeout}s)"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
wait_for_edge_text_since() {
local text="$1"
local start_line="$2"
local msg="$3"
local timeout="${4:-$EVENT_TIMEOUT}"
local deadline
local slice
deadline=$((SECONDS + timeout))
slice="$TMP_DIR/edge_wait_slice"
while true; do
tail -n +"$start_line" "$EDGE_OUT" > "$slice" || true
if grep -qF "$text" "$slice"; then
return 0
fi
ensure_smoke_processes_running "$msg"
if (( SECONDS >= deadline )); then
echo "[e2e] FAIL: timed out waiting for $msg (text: '$text', timeout: ${timeout}s)"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
send_cmd() {
LAST_CMD_START_LINE=$(($(wc -l < "$EDGE_OUT" | tr -d ' ') + 1))
echo "[e2e] > $1"
echo "$1" >&3
sleep "$COMMAND_SETTLE_SECONDS"
}
wait_for_node_registration
send_cmd "/nodes"
wait_for_edge_text_since "test-node (test-node)" "$LAST_CMD_START_LINE" "/nodes output"
send_cmd "/capabilities"
wait_for_edge_text_since "[node0-capabilities]" "$LAST_CMD_START_LINE" "/capabilities output"
wait_for_edge_text_since "targets = ${TARGET}" "$LAST_CMD_START_LINE" "/capabilities targets"
send_cmd "/transport"
wait_for_edge_text_since "[node0-transport]" "$LAST_CMD_START_LINE" "/transport output"
wait_for_edge_text_since "connected = true" "$LAST_CMD_START_LINE" "/transport connected status"
send_cmd "$FIRST_PROMPT"
wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 1 start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$FIRST_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 1 response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 1 completion" "$RUN_TIMEOUT"
send_cmd "$SECOND_PROMPT"
wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 2 start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$SECOND_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 2 response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 2 completion" "$RUN_TIMEOUT"
send_cmd "/session session2"
wait_for_edge_pattern_since "session .*session2" "$LAST_CMD_START_LINE" "/session acknowledgement"
send_cmd "/background on"
wait_for_edge_pattern_since "background .*on" "$LAST_CMD_START_LINE" "/background on acknowledgement"
send_cmd "$BACKGROUND_PROMPT"
wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=.*session=session2.*background=true" "$LAST_CMD_START_LINE" "background run start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$BACKGROUND_EXPECTED" "$LAST_CMD_START_LINE" "background run response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "background run completion" "$RUN_TIMEOUT"
send_cmd "/background off"
wait_for_edge_pattern_since "background .*off" "$LAST_CMD_START_LINE" "/background off acknowledgement"
send_cmd "/sessions"
wait_for_edge_text_since "[node0-sessions]" "$LAST_CMD_START_LINE" "/sessions output"
if [ "$IS_PERSISTENT" -eq 1 ]; then
send_cmd "/terminate-session"
wait_for_edge_text_since "terminated session" "$LAST_CMD_START_LINE" "/terminate-session acknowledgement"
fi
if [ "$HAS_STATUS" -eq 1 ]; then
send_cmd "/status"
wait_for_edge_text_since "[node0-status]" "$LAST_CMD_START_LINE" "/status output" "$STATUS_TIMEOUT"
fi
IDLE_FAIL=0
IDLE_MARKER_HITS=""
if [ "$IDLE_SECONDS" -gt 0 ]; then
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
NODE_BASELINE=$(wc -l < "$NODE_OUT" | tr -d ' ')
echo "[e2e] idle monitor: sleeping ${IDLE_SECONDS}s (edge baseline=${EDGE_BASELINE} node baseline=${NODE_BASELINE})"
sleep "$IDLE_SECONDS"
IDLE_PATTERN='transport_closed|heartbeat_timeout|node unregistered|disconnected from edge|transport_close_reason'
EDGE_IDLE_TAIL="$TMP_DIR/edge_idle_tail"
NODE_IDLE_TAIL="$TMP_DIR/node_idle_tail"
tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" > "$EDGE_IDLE_TAIL" || true
tail -n +"$((NODE_BASELINE + 1))" "$NODE_OUT" > "$NODE_IDLE_TAIL" || true
if grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL" >/dev/null 2>&1; then
IDLE_FAIL=1
IDLE_MARKER_HITS+=$'\n--- EDGE idle window markers ---\n'
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL")
fi
if grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL" >/dev/null 2>&1; then
IDLE_FAIL=1
IDLE_MARKER_HITS+=$'\n--- NODE idle window markers ---\n'
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL")
fi
fi
send_cmd "/exit"
exec 3>&-
wait $EDGE_PID || true
kill $NODE_PID 2>/dev/null || true
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
echo "==================="
FAIL=0
check_grep() {
local pattern="$1"
local file="$2"
local msg="$3"
if ! grep -q "$pattern" "$file"; then
echo "[e2e] FAIL: $msg (pattern: '$pattern')"
FAIL=1
fi
}
check_no_grep() {
local pattern="$1"
local file="$2"
local msg="$3"
if grep -q "$pattern" "$file"; then
echo "[e2e] FAIL: $msg (found pattern: '$pattern')"
grep "$pattern" "$file"
FAIL=1
fi
}
check_fail_markers() {
local file="$1"
# Strict failure detection for system/console errors
# We catch panics, unexpected exits, startup failures, and run-level error events.
if grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[[^]]+-evt\] error" "$file" >/dev/null; then
echo "[e2e] FAIL: detected error marker in $file"
grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[[^]]+-evt\] error" "$file"
FAIL=1
fi
}
check_node_messages_relayed_to_edge() {
local node_messages="$TMP_DIR/node_messages"
local edge_messages="$TMP_DIR/edge_messages"
local node_counts="$TMP_DIR/node_message_counts"
local count
local payload
local edge_count
# Compare the messages the node printed locally with the messages rendered
# by the edge console. A complete event is not enough if some node output
# never reached the edge.
awk '
/^\[node-message\] / {
in_message = 1
sub(/^\[node-message\] /, "")
if ($0 != "<empty>") {
print
}
next
}
in_message && /^\[/ {
in_message = 0
}
in_message && /^\{/ {
in_message = 0
}
in_message && $0 != "" {
print
}
' "$NODE_OUT" > "$node_messages"
awk '
/^\[[^]]+-msg\] / {
sub(/^\[[^]]+-msg\] /, "")
if ($0 != "<empty>") {
print
}
}
' "$EDGE_OUT" > "$edge_messages"
if [ ! -s "$node_messages" ]; then
echo "[e2e] FAIL: no node-generated message payloads found in node output"
FAIL=1
return
fi
sort "$node_messages" | uniq -c > "$node_counts"
while read -r count payload; do
if [ -z "$payload" ]; then
continue
fi
edge_count=$(grep -Fxc -- "$payload" "$edge_messages" || true)
if [ "$edge_count" -lt "$count" ]; then
echo "[e2e] FAIL: node message payload was not fully relayed to edge"
echo "[e2e] missing-or-short payload: $payload"
echo "[e2e] node_count=$count edge_count=$edge_count"
FAIL=1
fi
done < "$node_counts"
}
check_grep "test-node" "$EDGE_OUT" "node registration not found"
check_grep "start run_id=" "$EDGE_OUT" "run start not found"
check_grep "complete run_id=" "$EDGE_OUT" "run completion not found"
if [ "$(grep -c "\\[node0-evt\\] complete run_id=" "$EDGE_OUT" || true)" -lt 3 ]; then
echo "[e2e] FAIL: expected at least 3 completed runs (foreground x2 + background x1)"
FAIL=1
fi
check_grep "$FIRST_EXPECTED" "$EDGE_OUT" "foreground run 1 response not found"
check_grep "$SECOND_EXPECTED" "$EDGE_OUT" "foreground run 2 response not found"
check_grep "$BACKGROUND_EXPECTED" "$EDGE_OUT" "background run response not found"
if [ "$EXPECT_TAIL_MESSAGES" -eq 1 ]; then
check_grep "$FIRST_TAIL_EXPECTED" "$EDGE_OUT" "foreground run 1 tail response not found"
check_grep "$SECOND_TAIL_EXPECTED" "$EDGE_OUT" "foreground run 2 tail response not found"
check_grep "$BACKGROUND_TAIL_EXPECTED" "$EDGE_OUT" "background run tail response not found"
fi
check_no_grep "\\[node0-msg\\] <empty>" "$EDGE_OUT" "empty node message found"
check_node_messages_relayed_to_edge
check_grep "\\[node0-capabilities\\]" "$EDGE_OUT" "/capabilities output not found"
check_grep "targets = ${TARGET}" "$EDGE_OUT" "/capabilities targets not found"
check_grep "\\[node0-transport\\]" "$EDGE_OUT" "/transport output not found"
check_grep "connected = true" "$EDGE_OUT" "/transport connected status not found"
check_grep "\\[node0-sessions\\]" "$EDGE_OUT" "/sessions output not found"
if [ "$IS_PERSISTENT" -eq 1 ]; then
check_grep "persistent:${TARGET}/session2" "$EDGE_OUT" "/sessions entry for session2 not found"
fi
check_no_grep 'file:iop.db?cache=shared&mode=rwc' "$NODE_OUT" "node store used repo-root fallback dsn"
if [ "$IS_PERSISTENT" -eq 1 ]; then
check_grep "terminated session" "$EDGE_OUT" "/terminate-session success not found"
fi
check_fail_markers "$EDGE_OUT"
check_fail_markers "$NODE_OUT"
if [ "$IDLE_FAIL" -eq 1 ]; then
echo "[e2e] FAIL: idle window transport close markers detected"
echo "$IDLE_MARKER_HITS"
FAIL=1
fi
REAL_PROFILE_STATUS="N/A (mock used)"
if [ "$PROFILE" != "mock" ]; then
if [ $FAIL -eq 0 ]; then
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (command not found)"
FAIL=1
elif grep -qE "login|auth|permission|token|failed: EOF" "$NODE_OUT" && ! grep -q "complete run_id=" "$EDGE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (auth/login or startup failure)"
FAIL=1
else
REAL_PROFILE_STATUS="PASSED"
fi
else
# Try to distinguish between BLOCKED and FAILED for real profile
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (command not found)"
elif grep -qE "login|auth|permission|token" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (auth/login required)"
else
REAL_PROFILE_STATUS="FAILED"
fi
fi
echo "[e2e] Real profile ($PROFILE) status: $REAL_PROFILE_STATUS"
fi
if [ $FAIL -eq 1 ]; then
echo "[e2e] Auxiliary smoke test FAILED."
exit 1
fi
echo "[e2e] Auxiliary smoke test PASSED."
echo "[e2e] Completion still requires bin/edge.sh + bin/node.sh user-flow verification."
exit 0
Reference in a new issue View git blame Copy permalink