331 lines
12 KiB
Bash
Executable file
331 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# edge-node-reconnect-diagnostic.sh - repo-internal diagnostic for edge/node reconnect
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
echo "[diagnostic] Starting edge-node-reconnect-diagnostic (repo-internal)..."
|
|
|
|
TMP_DIR=$(mktemp -d /tmp/iop-reconnect-diag-XXXXXX)
|
|
|
|
kill_process_tree() {
|
|
local pid="$1"
|
|
local child
|
|
if [ -z "$pid" ]; then
|
|
return 0
|
|
fi
|
|
if command -v pgrep >/dev/null 2>&1; then
|
|
while IFS= read -r child; do
|
|
if [ -n "$child" ]; then
|
|
kill_process_tree "$child"
|
|
fi
|
|
done <<EOF_CHILDREN
|
|
$(pgrep -P "$pid" 2>/dev/null || true)
|
|
EOF_CHILDREN
|
|
fi
|
|
kill "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
cleanup() {
|
|
set +e
|
|
echo "[diagnostic] Cleaning up..."
|
|
kill_process_tree "${NODE_PID:-}"
|
|
kill_process_tree "${EDGE_PID:-}"
|
|
rm -rf "$TMP_DIR" 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
EDGE_CONFIG="$TMP_DIR/edge.yaml"
|
|
NODE_CONFIG="$TMP_DIR/node.yaml"
|
|
EDGE_OUT="$TMP_DIR/edge.log"
|
|
NODE_OUT="$TMP_DIR/node.log"
|
|
CONSOLE_FIFO="$TMP_DIR/console.fifo"
|
|
|
|
# Registration wait ceiling; honor the dev override used by the verification command.
|
|
BIND_TIMEOUT="${IOP_DEV_RECONNECT_BIND_TIMEOUT:-45}"
|
|
|
|
PORT=$((30000 + RANDOM % 10000))
|
|
BOOTSTRAP_PORT=$((20000 + RANDOM % 10000))
|
|
EDGE_METRICS_PORT=$((40000 + RANDOM % 10000))
|
|
NODE_METRICS_PORT=$((50000 + RANDOM % 10000))
|
|
|
|
# Create edge config
|
|
cat <<EOF > "$EDGE_CONFIG"
|
|
server:
|
|
listen: "127.0.0.1:$PORT"
|
|
metrics:
|
|
port: $EDGE_METRICS_PORT
|
|
bootstrap:
|
|
listen: "127.0.0.1:$BOOTSTRAP_PORT"
|
|
artifact_dir: "$TMP_DIR/artifacts"
|
|
nodes:
|
|
- id: test-node
|
|
alias: test-node
|
|
token: test-token
|
|
adapters:
|
|
mock:
|
|
enabled: true
|
|
console:
|
|
adapter: mock
|
|
target: mock-stream
|
|
session_id: diagnostic-correlation
|
|
EOF
|
|
|
|
# Create node config
|
|
cat <<EOF > "$NODE_CONFIG"
|
|
transport:
|
|
edge_addr: "127.0.0.1:$PORT"
|
|
token: test-token
|
|
reconnect:
|
|
interval_sec: 1
|
|
max_attempts: 0
|
|
metrics:
|
|
port: $NODE_METRICS_PORT
|
|
node:
|
|
id: test-node
|
|
alias: test-node
|
|
EOF
|
|
|
|
# Make console FIFO
|
|
mkfifo "$CONSOLE_FIFO"
|
|
|
|
# 1. Start edge
|
|
echo "[diagnostic] Starting edge.sh..."
|
|
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$CONSOLE_FIFO" > "$EDGE_OUT" 2>&1 &
|
|
EDGE_PID=$!
|
|
|
|
# Open FIFO for writing (blocks until edge reads it)
|
|
exec 3> "$CONSOLE_FIFO"
|
|
|
|
# 2. Start node
|
|
echo "[diagnostic] Starting node.sh..."
|
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 &
|
|
NODE_PID=$!
|
|
|
|
# Wait for node registration
|
|
echo "[diagnostic] Awaiting node registration..."
|
|
deadline=$((SECONDS + BIND_TIMEOUT))
|
|
while ! grep -q 'connected reason="registered"' "$EDGE_OUT" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[diagnostic] Timeout waiting for node registration" >&2
|
|
cat "$EDGE_OUT" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
echo "[diagnostic] Node registered"
|
|
|
|
# Wait for registration to settle
|
|
sleep 1
|
|
|
|
# Send /nodes command
|
|
echo "/nodes" >&3
|
|
sleep 0.5
|
|
|
|
# Send message 1
|
|
echo "Convert token IOP_E2E_HELLO_BASIC and reply only with converted token" >&3
|
|
# Wait for message 1 complete
|
|
deadline=$((SECONDS + 15))
|
|
while ! grep -q "complete run_id=" "$EDGE_OUT" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[diagnostic] Timeout waiting for message 1 completion" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
echo "[diagnostic] Message 1 completed"
|
|
|
|
# Send message 2
|
|
# Save baseline of edge log count to check next completion
|
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
|
echo "Convert token IOP_E2E_HELLO_FORMAL and reply only with converted token" >&3
|
|
# Wait for message 2 complete (new lines only)
|
|
deadline=$((SECONDS + 15))
|
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[diagnostic] Timeout waiting for message 2 completion" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
echo "[diagnostic] Message 2 completed"
|
|
|
|
# Send commands
|
|
echo "/capabilities" >&3
|
|
sleep 0.2
|
|
echo "/transport" >&3
|
|
sleep 0.2
|
|
sleep 0.5
|
|
|
|
# Restart Node
|
|
echo "[diagnostic] Killing node for reconnect test..."
|
|
kill_process_tree "$NODE_PID"
|
|
sleep 2
|
|
|
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
|
|
|
echo "[diagnostic] Restarting node..."
|
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" >> "$NODE_OUT" 2>&1 &
|
|
NODE_PID=$!
|
|
|
|
# Wait for re-registration (new lines only)
|
|
deadline=$((SECONDS + BIND_TIMEOUT))
|
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -E 'connected reason="registered"|node ready' 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[diagnostic] Timeout waiting for node reconnect" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
echo "[diagnostic] Node reconnected"
|
|
|
|
# Wait for reconnect to settle
|
|
sleep 1
|
|
|
|
# Send message 3 post-reconnect
|
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
|
echo "Convert token IOP_E2E_PING_BASIC and reply only with converted token" >&3
|
|
# Wait for message 3 complete
|
|
deadline=$((SECONDS + 10))
|
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do
|
|
if (( SECONDS >= deadline )); then
|
|
echo "[diagnostic] Timeout waiting for message 3 completion" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
echo "[diagnostic] Message 3 completed"
|
|
|
|
# Exit edge console
|
|
echo "/exit" >&3
|
|
exec 3>&-
|
|
|
|
# Wait for edge to finish
|
|
wait $EDGE_PID || true
|
|
|
|
echo "=== EDGE LOG ==="
|
|
cat "$EDGE_OUT"
|
|
echo "=== NODE LOG ==="
|
|
cat "$NODE_OUT"
|
|
|
|
# ---- Fail-fast verification of the direct two-process transcript ----
|
|
# Every branch below exits non-zero on a missing/extra token, a Node-vs-Edge
|
|
# payload mismatch, a mis-ordered or duplicated terminal event, a missing
|
|
# reconnect, or a missing command response. There is no silent skip: the previous
|
|
# version matched run_id on the [node0-msg] lines (which carry none), so its whole
|
|
# loop `continue`d and it printed PASS having compared nothing.
|
|
echo "[diagnostic] Verifying payload sequence, terminal ordering, and command responses..."
|
|
|
|
fail() {
|
|
printf '[diagnostic] VALIDATION FAILED: %b\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Runs are dispatched in this order: two before the reconnect, one after. The
|
|
# explicit mock provider echoes each fixed token in its deterministic stream.
|
|
EXPECTED_TOKENS=(IOP_E2E_HELLO_BASIC IOP_E2E_HELLO_FORMAL IOP_E2E_PING_BASIC)
|
|
|
|
# Edge-observed run ids, in order, from the authoritative start events.
|
|
mapfile -t RUN_IDS < <(grep -oE '\[node0-evt\] start run_id=[A-Za-z0-9_-]+' "$EDGE_OUT" | sed 's/.*run_id=//')
|
|
if [ "${#RUN_IDS[@]}" -ne "${#EXPECTED_TOKENS[@]}" ]; then
|
|
fail "expected ${#EXPECTED_TOKENS[@]} runs, edge start events show ${#RUN_IDS[@]}: ${RUN_IDS[*]:-<none>}"
|
|
fi
|
|
|
|
# Reconnect must have occurred: at least the initial + one reconnect registration.
|
|
CONNECTED_COUNT=$(grep -c 'connected reason="registered"' "$EDGE_OUT" || true)
|
|
if [ "${CONNECTED_COUNT:-0}" -lt 2 ]; then
|
|
fail "expected >=2 connected(registered) events (initial + reconnect), got ${CONNECTED_COUNT:-0}"
|
|
fi
|
|
|
|
for i in "${!RUN_IDS[@]}"; do
|
|
rid="${RUN_IDS[$i]}"
|
|
token="${EXPECTED_TOKENS[$i]}"
|
|
echo "[diagnostic] Checking run $((i + 1)) run_id=$rid token=$token"
|
|
|
|
start_ln=$(grep -nE "\[node0-evt\] start run_id=${rid}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true)
|
|
[ -n "$start_ln" ] || fail "run $rid: missing edge start event"
|
|
|
|
# The terminal event must appear exactly once for this run.
|
|
complete_ln=$(grep -nE "\[node0-evt\] complete run_id=${rid}( |\$)" "$EDGE_OUT" | cut -d: -f1 || true)
|
|
complete_count=$(printf '%s\n' "$complete_ln" | grep -c . || true)
|
|
[ "$complete_count" -eq 1 ] || fail "run $rid: expected exactly one complete event, got $complete_count"
|
|
[ "$complete_ln" -gt "$start_ln" ] || fail "run $rid: complete event precedes start"
|
|
|
|
# Payload lines strictly between this run's start and complete, in order.
|
|
run_payloads=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" \
|
|
| sed -n 's/^\[node0-msg\] //p' || true)
|
|
[ -n "$run_payloads" ] || fail "run $rid: no edge payload lines in trace"
|
|
printf '%s\n' "$run_payloads" | grep -Fq "$token" || fail "run $rid: edge payload does not contain fixed token $token"
|
|
|
|
# The terminal event must come strictly after the last payload line.
|
|
last_msg_rel=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" | grep -nE '\[node0-msg\]' | tail -n1 | cut -d: -f1 || true)
|
|
[ -n "$last_msg_rel" ] || fail "run $rid: no payload lines in trace"
|
|
last_msg_abs=$((start_ln + last_msg_rel - 1))
|
|
[ "$complete_ln" -gt "$last_msg_abs" ] || fail "run $rid: complete event is not after the last payload"
|
|
|
|
if [ "$i" -lt "$((${#RUN_IDS[@]} - 1))" ]; then
|
|
next_start_ln=$(grep -nE "\[node0-evt\] start run_id=${RUN_IDS[$((i + 1))]}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true)
|
|
else
|
|
next_start_ln=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
|
next_start_ln=$((next_start_ln + 1))
|
|
fi
|
|
[ -n "$next_start_ln" ] || fail "run $rid: missing next edge start boundary"
|
|
if sed -n "$((complete_ln + 1)),$((next_start_ln - 1))p" "$EDGE_OUT" | grep -qE '^\[node0-msg\] '; then
|
|
fail "run $rid: edge payload appeared after terminal event"
|
|
fi
|
|
|
|
# The Node process logs the same run with [node-event] start/complete bounds.
|
|
# Capture both the prefixed first payload line and any continuation lines
|
|
# emitted by a multiline adapter delta, then compare the sequence directly
|
|
# with the Edge relay for this exact run id.
|
|
node_start_ln=$(grep -nE "^\[node-event\] start run_id=${rid}\$" "$NODE_OUT" | cut -d: -f1 || true)
|
|
node_start_count=$(printf '%s\n' "$node_start_ln" | grep -c . || true)
|
|
[ "$node_start_count" -eq 1 ] || fail "run $rid: expected one Node start event, got $node_start_count"
|
|
node_complete_ln=$(grep -nE "^\[node-event\] complete run_id=${rid}( |\$)" "$NODE_OUT" | cut -d: -f1 || true)
|
|
node_complete_count=$(printf '%s\n' "$node_complete_ln" | grep -c . || true)
|
|
[ "$node_complete_count" -eq 1 ] || fail "run $rid: expected one Node complete event, got $node_complete_count"
|
|
[ "$node_complete_ln" -gt "$node_start_ln" ] || fail "run $rid: Node complete event precedes start"
|
|
|
|
node_payloads=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \
|
|
| sed -n 's/^\[node-message\] //p' || true)
|
|
if [ "${IOP_DEV_RECONNECT_INJECT_FAIL:-}" != "" ] && [ "$i" -eq "$((${#RUN_IDS[@]} - 1))" ]; then
|
|
# Test-only fault injection: drop the final Node payload from the last
|
|
# run so the actual per-run Node-vs-Edge comparison must fail-fast.
|
|
node_payloads=$(printf '%s\n' "$node_payloads" | sed '$d')
|
|
fi
|
|
if [ "$node_payloads" != "$run_payloads" ]; then
|
|
fail "run $rid: Node-vs-Edge payload sequence mismatch\n node: [$node_payloads]\n edge: [$run_payloads]"
|
|
fi
|
|
|
|
node_last_msg_rel=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \
|
|
| grep -nE '^\[node-message\] ' | tail -n1 | cut -d: -f1 || true)
|
|
[ -n "$node_last_msg_rel" ] || fail "run $rid: no Node payload lines in trace"
|
|
node_last_msg_abs=$((node_start_ln + node_last_msg_rel - 1))
|
|
[ "$node_complete_ln" -gt "$node_last_msg_abs" ] || fail "run $rid: Node complete event is not after the last payload"
|
|
if [ "$i" -lt "$((${#RUN_IDS[@]} - 1))" ]; then
|
|
next_node_start_ln=$(grep -nE "^\[node-event\] start run_id=${RUN_IDS[$((i + 1))]}\$" "$NODE_OUT" | head -n1 | cut -d: -f1 || true)
|
|
else
|
|
next_node_start_ln=$(wc -l < "$NODE_OUT" | tr -d ' ')
|
|
next_node_start_ln=$((next_node_start_ln + 1))
|
|
fi
|
|
[ -n "$next_node_start_ln" ] || fail "run $rid: missing next Node start boundary"
|
|
if sed -n "$((node_complete_ln + 1)),$((next_node_start_ln - 1))p" "$NODE_OUT" | grep -qE '^\[node-message\] '; then
|
|
fail "run $rid: Node payload appeared after terminal event"
|
|
fi
|
|
done
|
|
|
|
# Required command responses in the edge transcript (secret-safe: fixed markers only).
|
|
require_cmd() {
|
|
grep -qE "$2" "$EDGE_OUT" || fail "missing $1 command response"
|
|
}
|
|
require_cmd "/nodes" '^ node0 = test-node'
|
|
require_cmd "/capabilities" '\[node0-capabilities\]'
|
|
require_cmd "/transport" '\[node0-transport\]'
|
|
require_cmd "command banner" '^Commands: /nodes, /node <id|alias>, /session <id>, /background on|off, /capabilities, /transport, /exit$'
|
|
if grep -qE '/sessions|/status|/terminate-session' "$EDGE_OUT"; then
|
|
fail "removed session/status ownership command surface appeared in console transcript"
|
|
fi
|
|
|
|
echo "[diagnostic] PASS: ${#RUN_IDS[@]} mock-provider runs verified — registration, ordered Node==Edge payloads, one terminal after the last payload, /nodes, /capabilities, /transport, reconnect, and absent session/status ownership commands."
|