#!/usr/bin/env bash # edge-node-reconnect-diagnostic.sh - repo-internal diagnostic for edge/node reconnect set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" echo "[diagnostic] Starting edge-node-reconnect-diagnostic (repo-internal)..." TMP_DIR=$(mktemp -d /tmp/iop-reconnect-diag-XXXXXX) kill_process_tree() { local pid="$1" local child if [ -z "$pid" ]; then return 0 fi if command -v pgrep >/dev/null 2>&1; then while IFS= read -r child; do if [ -n "$child" ]; then kill_process_tree "$child" fi done </dev/null || true) EOF_CHILDREN fi kill "$pid" 2>/dev/null || true } cleanup() { set +e echo "[diagnostic] Cleaning up..." kill_process_tree "${NODE_PID:-}" kill_process_tree "${EDGE_PID:-}" rm -rf "$TMP_DIR" 2>/dev/null || true } trap cleanup EXIT EDGE_CONFIG="$TMP_DIR/edge.yaml" NODE_CONFIG="$TMP_DIR/node.yaml" EDGE_OUT="$TMP_DIR/edge.log" NODE_OUT="$TMP_DIR/node.log" CONSOLE_FIFO="$TMP_DIR/console.fifo" # Registration wait ceiling; honor the dev override used by the verification command. BIND_TIMEOUT="${IOP_DEV_RECONNECT_BIND_TIMEOUT:-45}" PORT=$((30000 + RANDOM % 10000)) BOOTSTRAP_PORT=$((20000 + RANDOM % 10000)) EDGE_METRICS_PORT=$((40000 + RANDOM % 10000)) NODE_METRICS_PORT=$((50000 + RANDOM % 10000)) # Create edge config cat < "$EDGE_CONFIG" server: listen: "127.0.0.1:$PORT" metrics: port: $EDGE_METRICS_PORT bootstrap: listen: "127.0.0.1:$BOOTSTRAP_PORT" artifact_dir: "$TMP_DIR/artifacts" nodes: - id: test-node alias: test-node token: test-token adapters: mock: enabled: true console: adapter: mock target: mock-stream session_id: diagnostic-correlation EOF # Create node config cat < "$NODE_CONFIG" transport: edge_addr: "127.0.0.1:$PORT" token: test-token reconnect: interval_sec: 1 max_attempts: 0 metrics: port: $NODE_METRICS_PORT node: id: test-node alias: test-node EOF # Make console FIFO mkfifo "$CONSOLE_FIFO" # 1. Start edge echo "[diagnostic] Starting edge.sh..." IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$CONSOLE_FIFO" > "$EDGE_OUT" 2>&1 & EDGE_PID=$! # Open FIFO for writing (blocks until edge reads it) exec 3> "$CONSOLE_FIFO" # 2. Start node echo "[diagnostic] Starting node.sh..." IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 & NODE_PID=$! # Wait for node registration echo "[diagnostic] Awaiting node registration..." deadline=$((SECONDS + BIND_TIMEOUT)) while ! grep -q 'connected reason="registered"' "$EDGE_OUT" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for node registration" >&2 cat "$EDGE_OUT" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Node registered" # Wait for registration to settle sleep 1 # Send /nodes command echo "/nodes" >&3 sleep 0.5 # Send message 1 echo "Convert token IOP_E2E_HELLO_BASIC and reply only with converted token" >&3 # Wait for message 1 complete deadline=$((SECONDS + 15)) while ! grep -q "complete run_id=" "$EDGE_OUT" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 1 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 1 completed" # Send message 2 # Save baseline of edge log count to check next completion EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "Convert token IOP_E2E_HELLO_FORMAL and reply only with converted token" >&3 # Wait for message 2 complete (new lines only) deadline=$((SECONDS + 15)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 2 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 2 completed" # Send commands echo "/capabilities" >&3 sleep 0.2 echo "/transport" >&3 sleep 0.2 sleep 0.5 # Restart Node echo "[diagnostic] Killing node for reconnect test..." kill_process_tree "$NODE_PID" sleep 2 EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "[diagnostic] Restarting node..." IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" >> "$NODE_OUT" 2>&1 & NODE_PID=$! # Wait for re-registration (new lines only) deadline=$((SECONDS + BIND_TIMEOUT)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -E 'connected reason="registered"|node ready' 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for node reconnect" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Node reconnected" # Wait for reconnect to settle sleep 1 # Send message 3 post-reconnect EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ') echo "Convert token IOP_E2E_PING_BASIC and reply only with converted token" >&3 # Wait for message 3 complete deadline=$((SECONDS + 10)) while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do if (( SECONDS >= deadline )); then echo "[diagnostic] Timeout waiting for message 3 completion" >&2 exit 1 fi sleep 0.5 done echo "[diagnostic] Message 3 completed" # Exit edge console echo "/exit" >&3 exec 3>&- # Wait for edge to finish wait $EDGE_PID || true echo "=== EDGE LOG ===" cat "$EDGE_OUT" echo "=== NODE LOG ===" cat "$NODE_OUT" # ---- Fail-fast verification of the direct two-process transcript ---- # Every branch below exits non-zero on a missing/extra token, a Node-vs-Edge # payload mismatch, a mis-ordered or duplicated terminal event, a missing # reconnect, or a missing command response. There is no silent skip: the previous # version matched run_id on the [node0-msg] lines (which carry none), so its whole # loop `continue`d and it printed PASS having compared nothing. echo "[diagnostic] Verifying payload sequence, terminal ordering, and command responses..." fail() { printf '[diagnostic] VALIDATION FAILED: %b\n' "$*" >&2 exit 1 } # Runs are dispatched in this order: two before the reconnect, one after. The # explicit mock provider echoes each fixed token in its deterministic stream. EXPECTED_TOKENS=(IOP_E2E_HELLO_BASIC IOP_E2E_HELLO_FORMAL IOP_E2E_PING_BASIC) # Edge-observed run ids, in order, from the authoritative start events. mapfile -t RUN_IDS < <(grep -oE '\[node0-evt\] start run_id=[A-Za-z0-9_-]+' "$EDGE_OUT" | sed 's/.*run_id=//') if [ "${#RUN_IDS[@]}" -ne "${#EXPECTED_TOKENS[@]}" ]; then fail "expected ${#EXPECTED_TOKENS[@]} runs, edge start events show ${#RUN_IDS[@]}: ${RUN_IDS[*]:-}" fi # Reconnect must have occurred: at least the initial + one reconnect registration. CONNECTED_COUNT=$(grep -c 'connected reason="registered"' "$EDGE_OUT" || true) if [ "${CONNECTED_COUNT:-0}" -lt 2 ]; then fail "expected >=2 connected(registered) events (initial + reconnect), got ${CONNECTED_COUNT:-0}" fi for i in "${!RUN_IDS[@]}"; do rid="${RUN_IDS[$i]}" token="${EXPECTED_TOKENS[$i]}" echo "[diagnostic] Checking run $((i + 1)) run_id=$rid token=$token" start_ln=$(grep -nE "\[node0-evt\] start run_id=${rid}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true) [ -n "$start_ln" ] || fail "run $rid: missing edge start event" # The terminal event must appear exactly once for this run. complete_ln=$(grep -nE "\[node0-evt\] complete run_id=${rid}( |\$)" "$EDGE_OUT" | cut -d: -f1 || true) complete_count=$(printf '%s\n' "$complete_ln" | grep -c . || true) [ "$complete_count" -eq 1 ] || fail "run $rid: expected exactly one complete event, got $complete_count" [ "$complete_ln" -gt "$start_ln" ] || fail "run $rid: complete event precedes start" # Payload lines strictly between this run's start and complete, in order. run_payloads=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" \ | sed -n 's/^\[node0-msg\] //p' || true) [ -n "$run_payloads" ] || fail "run $rid: no edge payload lines in trace" printf '%s\n' "$run_payloads" | grep -Fq "$token" || fail "run $rid: edge payload does not contain fixed token $token" # The terminal event must come strictly after the last payload line. last_msg_rel=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" | grep -nE '\[node0-msg\]' | tail -n1 | cut -d: -f1 || true) [ -n "$last_msg_rel" ] || fail "run $rid: no payload lines in trace" last_msg_abs=$((start_ln + last_msg_rel - 1)) [ "$complete_ln" -gt "$last_msg_abs" ] || fail "run $rid: complete event is not after the last payload" if [ "$i" -lt "$((${#RUN_IDS[@]} - 1))" ]; then next_start_ln=$(grep -nE "\[node0-evt\] start run_id=${RUN_IDS[$((i + 1))]}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true) else next_start_ln=$(wc -l < "$EDGE_OUT" | tr -d ' ') next_start_ln=$((next_start_ln + 1)) fi [ -n "$next_start_ln" ] || fail "run $rid: missing next edge start boundary" if sed -n "$((complete_ln + 1)),$((next_start_ln - 1))p" "$EDGE_OUT" | grep -qE '^\[node0-msg\] '; then fail "run $rid: edge payload appeared after terminal event" fi # The Node process logs the same run with [node-event] start/complete bounds. # Capture both the prefixed first payload line and any continuation lines # emitted by a multiline adapter delta, then compare the sequence directly # with the Edge relay for this exact run id. node_start_ln=$(grep -nE "^\[node-event\] start run_id=${rid}\$" "$NODE_OUT" | cut -d: -f1 || true) node_start_count=$(printf '%s\n' "$node_start_ln" | grep -c . || true) [ "$node_start_count" -eq 1 ] || fail "run $rid: expected one Node start event, got $node_start_count" node_complete_ln=$(grep -nE "^\[node-event\] complete run_id=${rid}( |\$)" "$NODE_OUT" | cut -d: -f1 || true) node_complete_count=$(printf '%s\n' "$node_complete_ln" | grep -c . || true) [ "$node_complete_count" -eq 1 ] || fail "run $rid: expected one Node complete event, got $node_complete_count" [ "$node_complete_ln" -gt "$node_start_ln" ] || fail "run $rid: Node complete event precedes start" node_payloads=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \ | sed -n 's/^\[node-message\] //p' || true) if [ "${IOP_DEV_RECONNECT_INJECT_FAIL:-}" != "" ] && [ "$i" -eq "$((${#RUN_IDS[@]} - 1))" ]; then # Test-only fault injection: drop the final Node payload from the last # run so the actual per-run Node-vs-Edge comparison must fail-fast. node_payloads=$(printf '%s\n' "$node_payloads" | sed '$d') fi if [ "$node_payloads" != "$run_payloads" ]; then fail "run $rid: Node-vs-Edge payload sequence mismatch\n node: [$node_payloads]\n edge: [$run_payloads]" fi node_last_msg_rel=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \ | grep -nE '^\[node-message\] ' | tail -n1 | cut -d: -f1 || true) [ -n "$node_last_msg_rel" ] || fail "run $rid: no Node payload lines in trace" node_last_msg_abs=$((node_start_ln + node_last_msg_rel - 1)) [ "$node_complete_ln" -gt "$node_last_msg_abs" ] || fail "run $rid: Node complete event is not after the last payload" if [ "$i" -lt "$((${#RUN_IDS[@]} - 1))" ]; then next_node_start_ln=$(grep -nE "^\[node-event\] start run_id=${RUN_IDS[$((i + 1))]}\$" "$NODE_OUT" | head -n1 | cut -d: -f1 || true) else next_node_start_ln=$(wc -l < "$NODE_OUT" | tr -d ' ') next_node_start_ln=$((next_node_start_ln + 1)) fi [ -n "$next_node_start_ln" ] || fail "run $rid: missing next Node start boundary" if sed -n "$((node_complete_ln + 1)),$((next_node_start_ln - 1))p" "$NODE_OUT" | grep -qE '^\[node-message\] '; then fail "run $rid: Node payload appeared after terminal event" fi done # Required command responses in the edge transcript (secret-safe: fixed markers only). require_cmd() { grep -qE "$2" "$EDGE_OUT" || fail "missing $1 command response" } require_cmd "/nodes" '^ node0 = test-node' require_cmd "/capabilities" '\[node0-capabilities\]' require_cmd "/transport" '\[node0-transport\]' require_cmd "command banner" '^Commands: /nodes, /node , /session , /background on|off, /capabilities, /transport, /exit$' if grep -qE '/sessions|/status|/terminate-session' "$EDGE_OUT"; then fail "removed session/status ownership command surface appeared in console transcript" fi echo "[diagnostic] PASS: ${#RUN_IDS[@]} mock-provider runs verified — registration, ordered Node==Edge payloads, one terminal after the last payload, /nodes, /capabilities, /transport, reconnect, and absent session/status ownership commands."