toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/scripts/e2e-smoke.sh
toki 1479949db0 feat(node/cli): improve persistent session handling and Claude terminal support 
- Add writePrompt with character-by-character typing for terminal profiles (2ms delay)
- Separate cliOutput into text and markerLine fields for precise completion matching
- Add consumeCompleteLines for buffered output parsing
- Add matchAny for multi-line completion marker checking
- Auto-accept Claude Code Bypass Permissions warning at startup
- Change PTY output reader from bufio.Scanner to ptmx.Read for raw byte output
- Improve drainUntilIdle with profile info and Claude bypass warning handling
- Add Claude status raw output fallback when usage parsing fails
- Add comprehensive blackbox tests for persistent sessions (21 test cases)
- Update e2e smoke test with proper wait functions, mock CLI script, and timeout variables
- Update edge.yaml console target from gemini to claude-tui, increase timeouts
- Update web app: add dev documentation, verify script, engines field, test message
2026-05-18 14:22:58 +09:00

456 lines
15 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
if command -v shellcheck >/dev/null 2>&1; then
echo "[e2e] running shellcheck..."
shellcheck "$0" "$REPO_ROOT/bin/edge.sh" "$REPO_ROOT/bin/node.sh"
else
echo "[e2e] shellcheck not found, skipping"
fi
TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR" 2>/dev/null || true' EXIT
EDGE_CONFIG="$TMP_DIR/edge.yaml"
NODE_CONFIG="$TMP_DIR/node.yaml"
PORT=$((30000 + RANDOM % 10000))
EDGE_METRICS_PORT=$((40000 + RANDOM % 10000))
NODE_METRICS_PORT=$((50000 + RANDOM % 10000))
PROFILE="${IOP_E2E_PROFILE:-mock}"
IDLE_SECONDS="${IOP_E2E_IDLE_SECONDS:-0}"
REGISTER_TIMEOUT="${IOP_E2E_REGISTER_TIMEOUT:-60}"
EVENT_TIMEOUT="${IOP_E2E_EVENT_TIMEOUT:-30}"
RUN_TIMEOUT="${IOP_E2E_RUN_TIMEOUT:-120}"
STATUS_TIMEOUT="${IOP_E2E_STATUS_TIMEOUT:-$RUN_TIMEOUT}"
COMMAND_SETTLE_SECONDS="${IOP_E2E_COMMAND_SETTLE_SECONDS:-0.2}"
IS_PERSISTENT=0
HAS_STATUS=0
FIRST_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_alpha"
FIRST_EXPECTED="SMOKE_ALPHA"
SECOND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_beta"
SECOND_EXPECTED="SMOKE_BETA"
BACKGROUND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_bg"
BACKGROUND_EXPECTED="SMOKE_BG"
if [ "$PROFILE" = "mock" ]; then
echo "[e2e] preparing honest mock smoke test (using scripted cli adapter)..."
IS_PERSISTENT=1
TARGET="fake-cli"
MOCK_CLI="$TMP_DIR/fake-cli.sh"
cat <<'EOF' > "$MOCK_CLI"
#!/usr/bin/env sh
while IFS= read -r line; do
case "$line" in
*iop_smoke_alpha*) printf 'IOP_SMOKE_ALPHA\n' ;;
*iop_smoke_beta*) printf 'IOP_SMOKE_BETA\n' ;;
*iop_smoke_bg*) printf 'IOP_SMOKE_BG\n' ;;
*) printf 'IOP_SMOKE_UNKNOWN\n' ;;
esac
done
EOF
chmod +x "$MOCK_CLI"
cat <<EOF > "$EDGE_CONFIG"
server:
listen: "127.0.0.1:$PORT"
metrics:
port: $EDGE_METRICS_PORT
nodes:
- id: test-node
alias: test-node
token: test-token
runtime:
workspace_root: "$TMP_DIR/workspace"
adapters:
mock:
enabled: true
cli:
enabled: true
profiles:
fake-cli:
command: "$MOCK_CLI"
persistent: true
response_idle_timeout_ms: 1000
console:
adapter: cli
target: fake-cli
session_id: default
EOF
else
echo "[e2e] preparing real profile smoke test (profile: $PROFILE)..."
TARGET="$PROFILE"
PROFILE_BLOCK=$(awk -v profile="$PROFILE" '
$0 ~ "^[[:space:]]+"profile":" {
found=1;
print $0;
indent=index($0, profile) - 1;
next;
}
found {
if ($0 ~ "^[[:space:]]*$") { print $0; next; }
curr_indent=0;
while(curr_indent < length($0) && substr($0, curr_indent+1, 1) == " ") curr_indent++;
if(curr_indent > indent) {
print $0;
} else {
exit;
}
}
' "$REPO_ROOT/configs/edge.yaml")
if [ -z "$PROFILE_BLOCK" ]; then
echo "[e2e] BLOCKED: Profile '$PROFILE' not found in configs/edge.yaml"
exit 1
fi
if echo "$PROFILE_BLOCK" | grep -q "persistent:[[:space:]]*true"; then
IS_PERSISTENT=1
fi
# Simple check for status support based on command name
if echo "$PROFILE_BLOCK" | grep -i -E "gemini|claude|codex" >/dev/null; then
HAS_STATUS=1
fi
cat <<EOF > "$EDGE_CONFIG"
server:
listen: "127.0.0.1:$PORT"
metrics:
port: $EDGE_METRICS_PORT
nodes:
- id: test-node
alias: test-node
token: test-token
runtime:
workspace_root: "$TMP_DIR/workspace"
adapters:
cli:
enabled: true
profiles:
$PROFILE_BLOCK
console:
adapter: cli
target: $PROFILE
session_id: default
EOF
fi
cat <<EOF > "$NODE_CONFIG"
transport:
edge_addr: "127.0.0.1:$PORT"
token: test-token
reconnect_interval: 1s
metrics:
port: $NODE_METRICS_PORT
node:
id: test-node
alias: test-node
EOF
echo "[e2e] starting smoke test (profile: $PROFILE, port: $PORT, persistent: $IS_PERSISTENT, has_status: $HAS_STATUS)"
EDGE_OUT="$TMP_DIR/edge_out"
NODE_OUT="$TMP_DIR/node_out"
rm -f "$TMP_DIR/edge_fifo"
mkfifo "$TMP_DIR/edge_fifo"
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/bin/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
EDGE_PID=$!
exec 3> "$TMP_DIR/edge_fifo"
deadline=$((SECONDS + 10))
while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/'"$PORT" 2>/dev/null; do
if (( SECONDS >= deadline )); then
echo "[e2e] edge failed to bind port $PORT"
cat "$EDGE_OUT"
exit 1
fi
sleep 0.5
done
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/bin/node.sh" > "$NODE_OUT" 2>&1 &
NODE_PID=$!
LAST_CMD_START_LINE=1
ensure_smoke_processes_running() {
local context="$1"
if ! kill -0 "$EDGE_PID" 2>/dev/null; then
echo "[e2e] edge exited while waiting for $context"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
exit 1
fi
if ! kill -0 "$NODE_PID" 2>/dev/null; then
echo "[e2e] node exited while waiting for $context"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
}
wait_for_node_registration() {
local deadline
deadline=$((SECONDS + REGISTER_TIMEOUT))
echo "[e2e] waiting for node registration (timeout: ${REGISTER_TIMEOUT}s)"
while ! grep -q '\[node-test-node-event\] connected reason="registered"' "$EDGE_OUT"; do
ensure_smoke_processes_running "node registration"
if (( SECONDS >= deadline )); then
echo "[e2e] node registration timed out"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
wait_for_edge_pattern_since() {
local pattern="$1"
local start_line="$2"
local msg="$3"
local timeout="${4:-$EVENT_TIMEOUT}"
local deadline
local slice
deadline=$((SECONDS + timeout))
slice="$TMP_DIR/edge_wait_slice"
while true; do
tail -n +"$start_line" "$EDGE_OUT" > "$slice" || true
if grep -qE "$pattern" "$slice"; then
return 0
fi
ensure_smoke_processes_running "$msg"
if (( SECONDS >= deadline )); then
echo "[e2e] FAIL: timed out waiting for $msg (pattern: '$pattern', timeout: ${timeout}s)"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
wait_for_edge_text_since() {
local text="$1"
local start_line="$2"
local msg="$3"
local timeout="${4:-$EVENT_TIMEOUT}"
local deadline
local slice
deadline=$((SECONDS + timeout))
slice="$TMP_DIR/edge_wait_slice"
while true; do
tail -n +"$start_line" "$EDGE_OUT" > "$slice" || true
if grep -qF "$text" "$slice"; then
return 0
fi
ensure_smoke_processes_running "$msg"
if (( SECONDS >= deadline )); then
echo "[e2e] FAIL: timed out waiting for $msg (text: '$text', timeout: ${timeout}s)"
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
exit 1
fi
sleep 0.5
done
}
send_cmd() {
LAST_CMD_START_LINE=$(($(wc -l < "$EDGE_OUT" | tr -d ' ') + 1))
echo "[e2e] > $1"
echo "$1" >&3
sleep "$COMMAND_SETTLE_SECONDS"
}
wait_for_node_registration
send_cmd "/nodes"
wait_for_edge_text_since "test-node (test-node)" "$LAST_CMD_START_LINE" "/nodes output"
send_cmd "/capabilities"
wait_for_edge_text_since "[node-test-node-capabilities]" "$LAST_CMD_START_LINE" "/capabilities output"
wait_for_edge_text_since "targets = ${TARGET}" "$LAST_CMD_START_LINE" "/capabilities targets"
send_cmd "/transport"
wait_for_edge_text_since "[node-test-node-transport]" "$LAST_CMD_START_LINE" "/transport output"
wait_for_edge_text_since "connected = true" "$LAST_CMD_START_LINE" "/transport connected status"
send_cmd "$FIRST_PROMPT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 1 start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$FIRST_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 1 response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 1 completion" "$RUN_TIMEOUT"
send_cmd "$SECOND_PROMPT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 2 start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$SECOND_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 2 response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 2 completion" "$RUN_TIMEOUT"
send_cmd "/session session2"
wait_for_edge_pattern_since "session .*session2" "$LAST_CMD_START_LINE" "/session acknowledgement"
send_cmd "/background on"
wait_for_edge_pattern_since "background .*on" "$LAST_CMD_START_LINE" "/background on acknowledgement"
send_cmd "$BACKGROUND_PROMPT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] start run_id=.*session=session2.*background=true" "$LAST_CMD_START_LINE" "background run start" "$RUN_TIMEOUT"
wait_for_edge_text_since "$BACKGROUND_EXPECTED" "$LAST_CMD_START_LINE" "background run response" "$RUN_TIMEOUT"
wait_for_edge_pattern_since "\\[node-test-node-event\\] complete run_id=" "$LAST_CMD_START_LINE" "background run completion" "$RUN_TIMEOUT"
send_cmd "/background off"
wait_for_edge_pattern_since "background .*off" "$LAST_CMD_START_LINE" "/background off acknowledgement"
send_cmd "/sessions"
wait_for_edge_text_since "[node-test-node-sessions]" "$LAST_CMD_START_LINE" "/sessions output"
if [ "$IS_PERSISTENT" -eq 1 ]; then
send_cmd "/terminate-session"
wait_for_edge_text_since "terminated session" "$LAST_CMD_START_LINE" "/terminate-session acknowledgement"
fi
if [ "$HAS_STATUS" -eq 1 ]; then
send_cmd "/status"
wait_for_edge_text_since "[node-test-node-status]" "$LAST_CMD_START_LINE" "/status output" "$STATUS_TIMEOUT"
fi
IDLE_FAIL=0
IDLE_MARKER_HITS=""
if [ "$IDLE_SECONDS" -gt 0 ]; then
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
NODE_BASELINE=$(wc -l < "$NODE_OUT" | tr -d ' ')
echo "[e2e] idle monitor: sleeping ${IDLE_SECONDS}s (edge baseline=${EDGE_BASELINE} node baseline=${NODE_BASELINE})"
sleep "$IDLE_SECONDS"
IDLE_PATTERN='transport_closed|heartbeat_timeout|node unregistered|disconnected from edge|transport_close_reason'
EDGE_IDLE_TAIL="$TMP_DIR/edge_idle_tail"
NODE_IDLE_TAIL="$TMP_DIR/node_idle_tail"
tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" > "$EDGE_IDLE_TAIL" || true
tail -n +"$((NODE_BASELINE + 1))" "$NODE_OUT" > "$NODE_IDLE_TAIL" || true
if grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL" >/dev/null 2>&1; then
IDLE_FAIL=1
IDLE_MARKER_HITS+=$'\n--- EDGE idle window markers ---\n'
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL")
fi
if grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL" >/dev/null 2>&1; then
IDLE_FAIL=1
IDLE_MARKER_HITS+=$'\n--- NODE idle window markers ---\n'
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL")
fi
fi
send_cmd "/exit"
exec 3>&-
wait $EDGE_PID || true
kill $NODE_PID 2>/dev/null || true
echo "=== EDGE OUTPUT ==="
cat "$EDGE_OUT"
echo "=== NODE OUTPUT ==="
cat "$NODE_OUT"
echo "==================="
FAIL=0
check_grep() {
local pattern="$1"
local file="$2"
local msg="$3"
if ! grep -q "$pattern" "$file"; then
echo "[e2e] FAIL: $msg (pattern: '$pattern')"
FAIL=1
fi
}
check_no_grep() {
local pattern="$1"
local file="$2"
local msg="$3"
if grep -q "$pattern" "$file"; then
echo "[e2e] FAIL: $msg (found pattern: '$pattern')"
grep "$pattern" "$file"
FAIL=1
fi
}
check_fail_markers() {
local file="$1"
# Strict failure detection for system/console errors
# We catch panics, unexpected exits, startup failures, and run-level error events.
if grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[node-.*-event\] error" "$file" >/dev/null; then
echo "[e2e] FAIL: detected error marker in $file"
grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[node-.*-event\] error" "$file"
FAIL=1
fi
}
check_grep "test-node" "$EDGE_OUT" "node registration not found"
check_grep "start run_id=" "$EDGE_OUT" "run start not found"
check_grep "complete run_id=" "$EDGE_OUT" "run completion not found"
if [ "$(grep -c "\\[node-test-node-event\\] complete run_id=" "$EDGE_OUT" || true)" -lt 3 ]; then
echo "[e2e] FAIL: expected at least 3 completed runs (foreground x2 + background x1)"
FAIL=1
fi
check_grep "$FIRST_EXPECTED" "$EDGE_OUT" "foreground run 1 response not found"
check_grep "$SECOND_EXPECTED" "$EDGE_OUT" "foreground run 2 response not found"
check_grep "$BACKGROUND_EXPECTED" "$EDGE_OUT" "background run response not found"
check_no_grep "\\[node-test-node-message\\] <empty>" "$EDGE_OUT" "empty node message found"
check_grep "\\[node-test-node-capabilities\\]" "$EDGE_OUT" "/capabilities output not found"
check_grep "targets = ${TARGET}" "$EDGE_OUT" "/capabilities targets not found"
check_grep "\\[node-test-node-transport\\]" "$EDGE_OUT" "/transport output not found"
check_grep "connected = true" "$EDGE_OUT" "/transport connected status not found"
check_grep "\\[node-test-node-sessions\\]" "$EDGE_OUT" "/sessions output not found"
if [ "$IS_PERSISTENT" -eq 1 ]; then
check_grep "persistent:${TARGET}/session2" "$EDGE_OUT" "/sessions entry for session2 not found"
fi
check_no_grep 'file:iop.db?cache=shared&mode=rwc' "$NODE_OUT" "node store used repo-root fallback dsn"
if [ "$IS_PERSISTENT" -eq 1 ]; then
check_grep "terminated session" "$EDGE_OUT" "/terminate-session success not found"
fi
check_fail_markers "$EDGE_OUT"
check_fail_markers "$NODE_OUT"
if [ "$IDLE_FAIL" -eq 1 ]; then
echo "[e2e] FAIL: idle window transport close markers detected"
echo "$IDLE_MARKER_HITS"
FAIL=1
fi
REAL_PROFILE_STATUS="N/A (mock used)"
if [ "$PROFILE" != "mock" ]; then
if [ $FAIL -eq 0 ]; then
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (command not found)"
FAIL=1
elif grep -qE "login|auth|permission|token|failed: EOF" "$NODE_OUT" && ! grep -q "complete run_id=" "$EDGE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (auth/login or startup failure)"
FAIL=1
else
REAL_PROFILE_STATUS="PASSED"
fi
else
# Try to distinguish between BLOCKED and FAILED for real profile
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (command not found)"
elif grep -qE "login|auth|permission|token" "$NODE_OUT"; then
REAL_PROFILE_STATUS="BLOCKED (auth/login required)"
else
REAL_PROFILE_STATUS="FAILED"
fi
fi
echo "[e2e] Real profile ($PROFILE) status: $REAL_PROFILE_STATUS"
fi
if [ $FAIL -eq 1 ]; then
echo "[e2e] Smoke test FAILED."
exit 1
fi
echo "[e2e] Smoke test PASSED."
exit 0
Reference in a new issue View git blame Copy permalink