#!/usr/bin/env bash # edge-node-reconnect-diagnostic.sh - repo-internal diagnostic for edge/node reconnect set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" echo "[diagnostic] Starting edge-node-reconnect-diagnostic (repo-internal)..." TMP_DIR=$(mktemp -d /tmp/iop-reconnect-diag-XXXXXX) kill_process_tree() { local pid="$1" local child if [ -z "$pid" ]; then return 0 fi if command -v pgrep >/dev/null 2>&1; then while IFS= read -r child; do if [ -n "$child" ]; then kill_process_tree "$child" fi done </dev/null || true) EOF_CHILDREN fi kill "$pid" 2>/dev/null || true } cleanup() { set +e echo "[diagnostic] Cleaning up..." kill_process_tree "${NODE_PID:-}" kill_process_tree "${EDGE_PID:-}" rm -rf "$TMP_DIR" 2>/dev/null || true } trap cleanup EXIT EDGE_CONFIG="$TMP_DIR/edge.yaml" NODE_CONFIG="$TMP_DIR/node.yaml" EDGE_OUT="$TMP_DIR/edge.log" NODE_OUT="$TMP_DIR/node.log" CONSOLE_FIFO="$TMP_DIR/console.fifo" # Registration wait ceiling; honor the dev override used by the verification command. BIND_TIMEOUT="${IOP_DEV_RECONNECT_BIND_TIMEOUT:-45}" PORT=$((30000 + RANDOM % 10000)) BOOTSTRAP_PORT=$((20000 + RANDOM % 10000)) EDGE_METRICS_PORT=$((40000 + RANDOM % 10000)) NODE_METRICS_PORT=$((50000 + RANDOM % 10000)) # Create fake-cli.sh. The Node's own [node-message] transcript below is the # node-side payload record used for the Edge relay comparison. MOCK_CLI="$TMP_DIR/fake-cli.sh" cat <<'EOF' > "$MOCK_CLI" #!/usr/bin/env sh while IFS= read -r line; do token=$(printf '%s\n' "$line" | sed -nE 's/.*(IOP_E2E_[A-Z0-9_]+).*/\1/p' | head -n 1) if [ -n "$token" ]; then printf '%s\n%s_TAIL\n' "$token" "$token" else printf 'IOP_E2E_UNKNOWN\n' fi done EOF chmod +x "$MOCK_CLI" # Create edge config cat < "$EDGE_CONFIG" server: listen: "127.0.0.1:$PORT" metrics: port: $EDGE_METRICS_PORT bootstrap: listen: "127.0.0.1:$BOOTSTRAP_PORT" artifact_dir: "$TMP_DIR/artifacts" nodes: - id: test-node alias: test-node token: test-token adapters: cli: enabled: true profiles: fake-cli: command: "$MOCK_CLI" persistent: true response_idle_timeout_ms: 1000 console: adapter: cli target: fake-cli session_id: default EOF # Create node config cat < "$NODE_CONFIG" transport: edge_addr: "127.0.0.1:$PORT" token: test-token reconnect: interval_sec: 1 max_attempts: 0 metrics: port: $NODE_METRICS_PORT node: id: test-node alias: test-node EOF # Make console FIFO mkfifo "$CONSOLE_FIFO" # 1. Start edge echo "[diagnostic] Starting edge.sh..." IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$CONSOLE_FIFO" > "$EDGE_OUT" 2>&1 & EDGE_PID=$! # Open FIFO for writing (blocks until edge reads it) exec 3> "$CONSOLE_FIFO" # 2. Start node echo "[diagnostic] Starting node.sh..." IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 & NODE_PID=$! # Wait for node registration echo "[diagnostic] Awaiting node registration..." deadline=$((SECONDS + BIND_TIMEOUT)) while ! grep -q 'connected reason="registered"' "$EDGE_OUT" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for node registration" >&2 cat "$EDGE_OUT" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Node registered" # Wait for registration to settle sleep 1 # Send /nodes command echo "/nodes" >&3 sleep 0.5 # Send message 1 echo "Convert token IOP_E2E_HELLO_BASIC and reply only with converted token" >&3 # Wait for message 1 complete deadline=$((SECONDS + 15)) while ! grep -q "complete run_id=" "$EDGE_OUT" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 1 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 1 completed" # Send message 2 # Save baseline of edge log count to check next completion EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "Convert token IOP_E2E_HELLO_FORMAL and reply only with converted token" >&3 # Wait for message 2 complete (new lines only) deadline=$((SECONDS + 15)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 2 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 2 completed" # Send commands echo "/capabilities" >&3 sleep 0.2 echo "/transport" >&3 sleep 0.2 echo "/sessions" >&3 sleep 0.2 echo "/terminate-session" >&3 sleep 0.5 # Restart Node echo "[diagnostic] Killing node for reconnect test..." kill_process_tree "$NODE_PID" sleep 2 EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "[diagnostic] Restarting node..." IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" >> "$NODE_OUT" 2>&1 & NODE_PID=$! # Wait for re-registration (new lines only) deadline=$((SECONDS + BIND_TIMEOUT)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -E 'connected reason="registered"|node ready' 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for node reconnect" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Node reconnected" # Wait for reconnect to settle sleep 1 # Send message 3 post-reconnect EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "Convert token IOP_E2E_PING_BASIC and reply only with converted token" >&3 # Wait for message 3 complete deadline=$((SECONDS + 10)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 3 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 3 completed" # Exit edge console echo "/exit" >&3 exec 3>&- # Wait for edge to finish wait $EDGE_PID || true echo "=== EDGE LOG ===" cat "$EDGE_OUT" echo "=== NODE LOG ===" cat "$NODE_OUT" # ---- Fail-fast verification of the direct two-process transcript ---- # Every branch below exits non-zero on a missing/extra token, a Node-vs-Edge # payload mismatch, a mis-ordered or duplicated terminal event, a missing # reconnect, or a missing command response. There is no silent skip: the previous # version matched run_id on the [node0-msg] lines (which carry none), so its whole # loop `continue`d and it printed PASS having compared nothing. echo "[diagnostic] Verifying payload sequence, terminal ordering, and command responses..." fail() { printf '[diagnostic] VALIDATION FAILED: %b\n' "$*" >&2 exit 1 } # Runs are dispatched in this order: two before the reconnect, one after. Each run # streams exactly the extracted token then that token with a _TAIL suffix. EXPECTED_TOKENS=(IOP_E2E_HELLO_BASIC IOP_E2E_HELLO_FORMAL IOP_E2E_PING_BASIC) # Edge-observed run ids, in order, from the authoritative start events. mapfile -t RUN_IDS < <(grep -oE '\[node0-evt\] start run_id=[A-Za-z0-9_-]+' "$EDGE_OUT" | sed 's/.*run_id=//') if [ "${#RUN_IDS[@]}" -ne "${#EXPECTED_TOKENS[@]}" ]; then fail "expected ${#EXPECTED_TOKENS[@]} runs, edge start events show ${#RUN_IDS[@]}: ${RUN_IDS[*]:-}" fi # Reconnect must have occurred: at least the initial + one reconnect registration. CONNECTED_COUNT=$(grep -c 'connected reason="registered"' "$EDGE_OUT" || true) if [ "${CONNECTED_COUNT:-0}" -lt 2 ]; then fail "expected >=2 connected(registered) events (initial + reconnect), got ${CONNECTED_COUNT:-0}" fi for i in "${!RUN_IDS[@]}"; do rid="${RUN_IDS[$i]}" token="${EXPECTED_TOKENS[$i]}" echo "[diagnostic] Checking run $((i + 1)) run_id=$rid token=$token" start_ln=$(grep -nE "\[node0-evt\] start run_id=${rid}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true) [ -n "$start_ln" ] || fail "run $rid: missing edge start event" # The terminal event must appear exactly once for this run. complete_ln=$(grep -nE "\[node0-evt\] complete run_id=${rid}( |\$)" "$EDGE_OUT" | cut -d: -f1 || true) complete_count=$(printf '%s\n' "$complete_ln" | grep -c . || true) [ "$complete_count" -eq 1 ] || fail "run $rid: expected exactly one complete event, got $complete_count" [ "$complete_ln" -gt "$start_ln" ] || fail "run $rid: complete event precedes start" # Payload tokens strictly between this run's start and complete, in order. run_payloads=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" \ | grep -oE '\[node0-msg\] IOP_E2E_[A-Z0-9_]+' | sed 's/.*\] //' || true) want_payloads=$(printf '%s\n%s_TAIL' "$token" "$token") if [ "$run_payloads" != "$want_payloads" ]; then fail "run $rid ($token): edge payload sequence mismatch\n got: [$run_payloads]\n want: [$want_payloads]" fi # The terminal event must come strictly after the last payload line. last_msg_rel=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" | grep -nE '\[node0-msg\]' | tail -n1 | cut -d: -f1 || true) [ -n "$last_msg_rel" ] || fail "run $rid: no payload lines in trace" last_msg_abs=$((start_ln + last_msg_rel - 1)) [ "$complete_ln" -gt "$last_msg_abs" ] || fail "run $rid: complete event is not after the last payload" # The Node process logs the same run with [node-event] start/complete bounds. # Capture both the prefixed first payload line and any continuation lines # emitted by a multiline adapter delta, then compare the sequence directly # with the Edge relay for this exact run id. node_start_ln=$(grep -nE "^\[node-event\] start run_id=${rid}\$" "$NODE_OUT" | cut -d: -f1 || true) node_start_count=$(printf '%s\n' "$node_start_ln" | grep -c . || true) [ "$node_start_count" -eq 1 ] || fail "run $rid: expected one Node start event, got $node_start_count" node_complete_ln=$(grep -nE "^\[node-event\] complete run_id=${rid}( |\$)" "$NODE_OUT" | cut -d: -f1 || true) node_complete_count=$(printf '%s\n' "$node_complete_ln" | grep -c . || true) [ "$node_complete_count" -eq 1 ] || fail "run $rid: expected one Node complete event, got $node_complete_count" [ "$node_complete_ln" -gt "$node_start_ln" ] || fail "run $rid: Node complete event precedes start" node_payloads=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \ | sed -nE '/^\[node-message\] IOP_E2E_[A-Z0-9_]+$/{s/^\[node-message\] //;p;b}; /^IOP_E2E_[A-Z0-9_]+$/p' || true) if [ "${IOP_DEV_RECONNECT_INJECT_FAIL:-}" != "" ] && [ "$i" -eq "$((${#RUN_IDS[@]} - 1))" ]; then # Test-only fault injection: drop the final Node payload from the last # run so the actual per-run Node-vs-Edge comparison must fail-fast. node_payloads=$(printf '%s\n' "$node_payloads" | sed '$d') fi if [ "$node_payloads" != "$run_payloads" ]; then fail "run $rid: Node-vs-Edge payload sequence mismatch\n node: [$node_payloads]\n edge: [$run_payloads]" fi node_last_msg_rel=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \ | grep -nE '^\[node-message\] |^IOP_E2E_' | tail -n1 | cut -d: -f1 || true) [ -n "$node_last_msg_rel" ] || fail "run $rid: no Node payload lines in trace" node_last_msg_abs=$((node_start_ln + node_last_msg_rel - 1)) [ "$node_complete_ln" -gt "$node_last_msg_abs" ] || fail "run $rid: Node complete event is not after the last payload" done # Required command responses in the edge transcript (secret-safe: fixed markers only). require_cmd() { grep -qE "$2" "$EDGE_OUT" || fail "missing $1 command response" } require_cmd "/nodes" '^ node0 = test-node' require_cmd "/capabilities" '\[node0-capabilities\]' require_cmd "/transport" '\[node0-transport\]' require_cmd "/sessions" '\[node0-sessions\]' require_cmd "/terminate-session" 'terminated session .* node=node0' echo "[diagnostic] PASS: ${#RUN_IDS[@]} runs verified — payload sequence, one terminal after the last payload, Node==Edge; reconnect observed; all five command responses present."