toki
96e647984d
- Add EDGE_METRICS_PORT and NODE_METRICS_PORT to e2e smoke test configs - Remove '|| true' from shellcheck to fail on lint errors - Delete iop.db (generated test artifact)
309 lines
8.9 KiB
Bash
Executable file
309 lines
8.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
if command -v shellcheck >/dev/null 2>&1; then
|
|
echo "[e2e] running shellcheck..."
|
|
shellcheck "$0" "$REPO_ROOT/bin/edge.sh" "$REPO_ROOT/bin/node.sh"
|
|
else
|
|
echo "[e2e] shellcheck not found, skipping"
|
|
fi
|
|
|
|
TMP_DIR=$(mktemp -d)
|
|
trap 'rm -rf "$TMP_DIR" 2>/dev/null || true' EXIT
|
|
|
|
EDGE_CONFIG="$TMP_DIR/edge.yaml"
|
|
NODE_CONFIG="$TMP_DIR/node.yaml"
|
|
|
|
PORT=$((30000 + RANDOM % 10000))
|
|
EDGE_METRICS_PORT=$((40000 + RANDOM % 10000))
|
|
NODE_METRICS_PORT=$((50000 + RANDOM % 10000))
|
|
PROFILE="${IOP_E2E_PROFILE:-mock}"
|
|
IDLE_SECONDS="${IOP_E2E_IDLE_SECONDS:-0}"
|
|
IS_PERSISTENT=0
|
|
HAS_STATUS=0
|
|
|
|
if [ "$PROFILE" = "mock" ]; then
|
|
echo "[e2e] preparing honest mock smoke test (using cli adapter + cat)..."
|
|
IS_PERSISTENT=1
|
|
TARGET="fake-cat"
|
|
cat <<EOF > "$EDGE_CONFIG"
|
|
server:
|
|
listen: "127.0.0.1:$PORT"
|
|
metrics:
|
|
port: $EDGE_METRICS_PORT
|
|
nodes:
|
|
- id: test-node
|
|
alias: test-node
|
|
token: test-token
|
|
runtime:
|
|
workspace_root: "$TMP_DIR/workspace"
|
|
adapters:
|
|
mock:
|
|
enabled: true
|
|
cli:
|
|
enabled: true
|
|
profiles:
|
|
fake-cat:
|
|
command: "cat"
|
|
persistent: true
|
|
response_idle_timeout_ms: 1000
|
|
console:
|
|
adapter: cli
|
|
target: fake-cat
|
|
session_id: default
|
|
EOF
|
|
else
|
|
echo "[e2e] preparing real profile smoke test (profile: $PROFILE)..."
|
|
TARGET="$PROFILE"
|
|
PROFILE_BLOCK=$(awk -v profile="$PROFILE" '
|
|
$0 ~ "^[[:space:]]+"profile":" {
|
|
found=1;
|
|
print $0;
|
|
indent=index($0, profile) - 1;
|
|
next;
|
|
}
|
|
found {
|
|
if ($0 ~ "^[[:space:]]*$") { print $0; next; }
|
|
curr_indent=0;
|
|
while(curr_indent < length($0) && substr($0, curr_indent+1, 1) == " ") curr_indent++;
|
|
if(curr_indent > indent) {
|
|
print $0;
|
|
} else {
|
|
exit;
|
|
}
|
|
}
|
|
' "$REPO_ROOT/configs/edge.yaml")
|
|
|
|
if [ -z "$PROFILE_BLOCK" ]; then
|
|
echo "[e2e] BLOCKED: Profile '$PROFILE' not found in configs/edge.yaml"
|
|
exit 1
|
|
fi
|
|
|
|
if echo "$PROFILE_BLOCK" | grep -q "persistent:[[:space:]]*true"; then
|
|
IS_PERSISTENT=1
|
|
fi
|
|
|
|
# Simple check for status support based on command name
|
|
if echo "$PROFILE_BLOCK" | grep -i -E "gemini|claude|codex" >/dev/null; then
|
|
HAS_STATUS=1
|
|
fi
|
|
|
|
cat <<EOF > "$EDGE_CONFIG"
|
|
server:
|
|
listen: "127.0.0.1:$PORT"
|
|
metrics:
|
|
port: $EDGE_METRICS_PORT
|
|
nodes:
|
|
- id: test-node
|
|
alias: test-node
|
|
token: test-token
|
|
runtime:
|
|
workspace_root: "$TMP_DIR/workspace"
|
|
adapters:
|
|
cli:
|
|
enabled: true
|
|
profiles:
|
|
$PROFILE_BLOCK
|
|
console:
|
|
adapter: cli
|
|
target: $PROFILE
|
|
session_id: default
|
|
EOF
|
|
fi
|
|
|
|
cat <<EOF > "$NODE_CONFIG"
|
|
transport:
|
|
edge_addr: "127.0.0.1:$PORT"
|
|
token: test-token
|
|
reconnect_interval: 1s
|
|
metrics:
|
|
port: $NODE_METRICS_PORT
|
|
node:
|
|
id: test-node
|
|
alias: test-node
|
|
EOF
|
|
|
|
echo "[e2e] starting smoke test (profile: $PROFILE, port: $PORT, persistent: $IS_PERSISTENT, has_status: $HAS_STATUS)"
|
|
|
|
EDGE_OUT="$TMP_DIR/edge_out"
|
|
NODE_OUT="$TMP_DIR/node_out"
|
|
rm -f "$TMP_DIR/edge_fifo"
|
|
mkfifo "$TMP_DIR/edge_fifo"
|
|
|
|
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/bin/edge.sh" < "$TMP_DIR/edge_fifo" > "$EDGE_OUT" 2>&1 &
|
|
EDGE_PID=$!
|
|
exec 3> "$TMP_DIR/edge_fifo"
|
|
|
|
deadline=$((SECONDS + 10))
|
|
while ! timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/'"$PORT" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[e2e] edge failed to bind port $PORT"
|
|
cat "$EDGE_OUT"
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/bin/node.sh" > "$NODE_OUT" 2>&1 &
|
|
NODE_PID=$!
|
|
|
|
send_cmd() {
|
|
echo "[e2e] > $1"
|
|
echo "$1" >&3
|
|
sleep 3
|
|
}
|
|
|
|
STRICT_PROMPT="Reply with exactly 'OK' and do not inspect files or use any tools."
|
|
|
|
sleep 3
|
|
send_cmd "/nodes"
|
|
send_cmd "/capabilities"
|
|
send_cmd "/transport"
|
|
send_cmd "$STRICT_PROMPT"
|
|
send_cmd "/session session2"
|
|
send_cmd "/background on"
|
|
send_cmd "Reply with exactly 'OK' again."
|
|
send_cmd "/background off"
|
|
send_cmd "/sessions"
|
|
|
|
if [ "$IS_PERSISTENT" -eq 1 ]; then
|
|
send_cmd "/terminate-session"
|
|
fi
|
|
|
|
if [ "$HAS_STATUS" -eq 1 ]; then
|
|
send_cmd "/status"
|
|
fi
|
|
|
|
IDLE_FAIL=0
|
|
IDLE_MARKER_HITS=""
|
|
if [ "$IDLE_SECONDS" -gt 0 ]; then
|
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
|
NODE_BASELINE=$(wc -l < "$NODE_OUT" | tr -d ' ')
|
|
echo "[e2e] idle monitor: sleeping ${IDLE_SECONDS}s (edge baseline=${EDGE_BASELINE} node baseline=${NODE_BASELINE})"
|
|
sleep "$IDLE_SECONDS"
|
|
IDLE_PATTERN='transport_closed|heartbeat_timeout|node unregistered|disconnected from edge|transport_close_reason'
|
|
EDGE_IDLE_TAIL="$TMP_DIR/edge_idle_tail"
|
|
NODE_IDLE_TAIL="$TMP_DIR/node_idle_tail"
|
|
tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" > "$EDGE_IDLE_TAIL" || true
|
|
tail -n +"$((NODE_BASELINE + 1))" "$NODE_OUT" > "$NODE_IDLE_TAIL" || true
|
|
if grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL" >/dev/null 2>&1; then
|
|
IDLE_FAIL=1
|
|
IDLE_MARKER_HITS+=$'\n--- EDGE idle window markers ---\n'
|
|
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$EDGE_IDLE_TAIL")
|
|
fi
|
|
if grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL" >/dev/null 2>&1; then
|
|
IDLE_FAIL=1
|
|
IDLE_MARKER_HITS+=$'\n--- NODE idle window markers ---\n'
|
|
IDLE_MARKER_HITS+=$(grep -E "$IDLE_PATTERN" "$NODE_IDLE_TAIL")
|
|
fi
|
|
fi
|
|
|
|
send_cmd "/exit"
|
|
exec 3>&-
|
|
|
|
wait $EDGE_PID || true
|
|
kill $NODE_PID 2>/dev/null || true
|
|
|
|
echo "=== EDGE OUTPUT ==="
|
|
cat "$EDGE_OUT"
|
|
echo "=== NODE OUTPUT ==="
|
|
cat "$NODE_OUT"
|
|
echo "==================="
|
|
|
|
FAIL=0
|
|
check_grep() {
|
|
local pattern="$1"
|
|
local file="$2"
|
|
local msg="$3"
|
|
if ! grep -q "$pattern" "$file"; then
|
|
echo "[e2e] FAIL: $msg (pattern: '$pattern')"
|
|
FAIL=1
|
|
fi
|
|
}
|
|
|
|
check_no_grep() {
|
|
local pattern="$1"
|
|
local file="$2"
|
|
local msg="$3"
|
|
if grep -q "$pattern" "$file"; then
|
|
echo "[e2e] FAIL: $msg (found pattern: '$pattern')"
|
|
grep "$pattern" "$file"
|
|
FAIL=1
|
|
fi
|
|
}
|
|
|
|
check_fail_markers() {
|
|
local file="$1"
|
|
# Strict failure detection for system/console errors
|
|
# We catch panics, unexpected exits, startup failures, and run-level error events.
|
|
if grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[node-.*-event\] error" "$file" >/dev/null; then
|
|
echo "[e2e] FAIL: detected error marker in $file"
|
|
grep -i -E "panic:|unexpected|startup failed|cancel error|node reported error|does not support|adapter .* not found|no session|error run_id=|\[node-event\] error|\[node-.*-event\] error" "$file"
|
|
FAIL=1
|
|
fi
|
|
}
|
|
|
|
check_grep "test-node" "$EDGE_OUT" "node registration not found"
|
|
check_grep "start run_id=" "$EDGE_OUT" "run start not found"
|
|
check_grep "complete run_id=" "$EDGE_OUT" "run completion not found"
|
|
|
|
check_grep "\\[node-test-node-capabilities\\]" "$EDGE_OUT" "/capabilities output not found"
|
|
check_grep "targets = ${TARGET}" "$EDGE_OUT" "/capabilities targets not found"
|
|
check_grep "\\[node-test-node-transport\\]" "$EDGE_OUT" "/transport output not found"
|
|
check_grep "connected = true" "$EDGE_OUT" "/transport connected status not found"
|
|
check_grep "\\[node-test-node-sessions\\]" "$EDGE_OUT" "/sessions output not found"
|
|
|
|
if [ "$IS_PERSISTENT" -eq 1 ]; then
|
|
check_grep "persistent:${TARGET}/session2" "$EDGE_OUT" "/sessions entry for session2 not found"
|
|
fi
|
|
|
|
check_no_grep 'file:iop.db?cache=shared&mode=rwc' "$NODE_OUT" "node store used repo-root fallback dsn"
|
|
|
|
if [ "$IS_PERSISTENT" -eq 1 ]; then
|
|
check_grep "terminated session" "$EDGE_OUT" "/terminate-session success not found"
|
|
fi
|
|
|
|
check_fail_markers "$EDGE_OUT"
|
|
check_fail_markers "$NODE_OUT"
|
|
|
|
if [ "$IDLE_FAIL" -eq 1 ]; then
|
|
echo "[e2e] FAIL: idle window transport close markers detected"
|
|
echo "$IDLE_MARKER_HITS"
|
|
FAIL=1
|
|
fi
|
|
|
|
REAL_PROFILE_STATUS="N/A (mock used)"
|
|
if [ "$PROFILE" != "mock" ]; then
|
|
if [ $FAIL -eq 0 ]; then
|
|
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
|
|
REAL_PROFILE_STATUS="BLOCKED (command not found)"
|
|
FAIL=1
|
|
elif grep -qE "login|auth|permission|token|failed: EOF" "$NODE_OUT" && ! grep -q "complete run_id=" "$EDGE_OUT"; then
|
|
REAL_PROFILE_STATUS="BLOCKED (auth/login or startup failure)"
|
|
FAIL=1
|
|
else
|
|
REAL_PROFILE_STATUS="PASSED"
|
|
fi
|
|
else
|
|
# Try to distinguish between BLOCKED and FAILED for real profile
|
|
if grep -q -E "command not found|no such file" "$NODE_OUT"; then
|
|
REAL_PROFILE_STATUS="BLOCKED (command not found)"
|
|
elif grep -qE "login|auth|permission|token" "$NODE_OUT"; then
|
|
REAL_PROFILE_STATUS="BLOCKED (auth/login required)"
|
|
else
|
|
REAL_PROFILE_STATUS="FAILED"
|
|
fi
|
|
fi
|
|
echo "[e2e] Real profile ($PROFILE) status: $REAL_PROFILE_STATUS"
|
|
fi
|
|
|
|
if [ $FAIL -eq 1 ]; then
|
|
echo "[e2e] Smoke test FAILED."
|
|
exit 1
|
|
fi
|
|
|
|
echo "[e2e] Smoke test PASSED."
|
|
exit 0
|