207 lines
7.2 KiB
Bash
Executable file
207 lines
7.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: run_stress.sh [--quick|--full] [--profile a,b,...]
|
|
|
|
Runs the Proto Socket inbound-queue-ordering stress / benchmark harness and writes a
|
|
local result record under agent-test/runs/.
|
|
|
|
Modes:
|
|
--quick small, fast baseline for smoke/regression (default)
|
|
--full Milestone baseline sizes (1k/10k burst, 30s sustained, 16 parallel clients)
|
|
|
|
Profiles (comma separated, default: all):
|
|
roundtrip request-response latency at concurrency 1/16/64/256
|
|
burst single-connection fire-and-forget burst ordering / leak check
|
|
sustained time-boxed sustained request-response with peak memory
|
|
parallel multi-connection FIFO / nonce isolation
|
|
gateway worker_threads gateway (on) vs inline decode fallback (off) baseline
|
|
|
|
Stability pass criteria (hard fail): timeout / nonce mismatch / response type mismatch /
|
|
per-connection FIFO violation / pending-queue leak all 0. Absolute throughput and latency
|
|
are stored as an environment-dependent baseline, not a fixed pass threshold.
|
|
USAGE
|
|
}
|
|
|
|
mode="quick"
|
|
profiles=""
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--quick) mode="quick" ;;
|
|
--full) mode="full" ;;
|
|
--mode=*) mode="${1#--mode=}" ;;
|
|
--profile=*) profiles="${1#--profile=}" ;;
|
|
--profiles=*) profiles="${1#--profiles=}" ;;
|
|
--profile|--profiles)
|
|
shift
|
|
profiles="${1:-}"
|
|
;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
case "$mode" in
|
|
quick|full) ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
repo_root="$(cd "$script_dir/../../../../.." && pwd)"
|
|
ts_dir="$repo_root/typescript"
|
|
tsx_bin="$ts_dir/node_modules/.bin/tsx"
|
|
|
|
run_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
run_record_stamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
record_profile="proto-socket-stress-${mode}"
|
|
git_ref="$(git -C "$repo_root" rev-parse --short HEAD 2>/dev/null || echo "unknown")"
|
|
|
|
record_dir="$repo_root/agent-test/runs"
|
|
mkdir -p "$record_dir"
|
|
record_file="$record_dir/${run_record_stamp}-${record_profile}.md"
|
|
|
|
harness_args=("--mode=$mode")
|
|
display_profiles="all"
|
|
if [ -n "$profiles" ]; then
|
|
harness_args+=("--profiles=$profiles")
|
|
display_profiles="$profiles"
|
|
fi
|
|
|
|
run_cmd="bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_stress.sh --$mode"
|
|
if [ -n "$profiles" ]; then
|
|
run_cmd="$run_cmd --profile $profiles"
|
|
fi
|
|
|
|
stdout_file="$(mktemp "${TMPDIR:-/tmp}/proto-socket-stress.out.XXXXXX")"
|
|
stderr_file="$(mktemp "${TMPDIR:-/tmp}/proto-socket-stress.err.XXXXXX")"
|
|
cleanup() {
|
|
rm -f "$stdout_file" "$stderr_file"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# tsx runner가 없으면 PASS로 기록하지 않고 BLOCKED로 보고한다.
|
|
if [ ! -x "$tsx_bin" ]; then
|
|
overall_result="BLOCKED"
|
|
block_reason="tsx runner not found at $tsx_bin (run 'npm install' in typescript/)"
|
|
{
|
|
printf '%s\n' '---'
|
|
printf 'test_env: local\n'
|
|
printf 'record_type: stress-result\n'
|
|
printf 'test_profile: %s\n' "$record_profile"
|
|
printf 'created_at: %s\n' "$run_started_at"
|
|
printf 'overall_result: %s\n' "$overall_result"
|
|
printf '%s\n\n' '---'
|
|
printf '# %s local 결과 기록\n\n' "$record_profile"
|
|
printf '## 실행 정보\n\n'
|
|
printf -- '- 실행 일시: %s\n' "$run_started_at"
|
|
printf -- '- git ref: %s\n' "$git_ref"
|
|
printf -- '- 실행 명령: `%s`\n' "$run_cmd"
|
|
printf -- '- mode: %s\n' "$mode"
|
|
printf -- '- profiles: %s\n' "$display_profiles"
|
|
printf -- '- 전체 결과값: %s\n\n' "$overall_result"
|
|
printf '## 차단\n\n'
|
|
printf -- '- 차단 사유: %s\n' "$block_reason"
|
|
} >"$record_file"
|
|
printf '%s\n' "$block_reason" >&2
|
|
printf '\n결과 기록 파일: `%s`\n' "${record_file#$repo_root/}"
|
|
exit 3
|
|
fi
|
|
|
|
printf 'RUN stress mode=%s profiles=%s\n' "$mode" "$display_profiles" >&2
|
|
|
|
set +e
|
|
(
|
|
cd "$ts_dir" &&
|
|
"$tsx_bin" bench/stress.ts "${harness_args[@]}"
|
|
) >"$stdout_file" 2>"$stderr_file"
|
|
exit_code=$?
|
|
set -e
|
|
|
|
# stderr 진행 로그를 그대로 흘려 사용자가 실행 흐름을 볼 수 있게 한다.
|
|
cat "$stderr_file" >&2
|
|
|
|
summary_line="$(grep '^SUMMARY|' "$stdout_file" | tail -n 1 || true)"
|
|
overall_result="FAIL"
|
|
stability_violations="?"
|
|
if [ -n "$summary_line" ]; then
|
|
status_field="$(printf '%s' "$summary_line" | tr '|' '\n' | sed -n 's/^status=//p' | head -n 1)"
|
|
violations_field="$(printf '%s' "$summary_line" | tr '|' '\n' | sed -n 's/^stability_violations=//p' | head -n 1)"
|
|
if [ -n "$status_field" ]; then
|
|
overall_result="$status_field"
|
|
fi
|
|
if [ -n "$violations_field" ]; then
|
|
stability_violations="$violations_field"
|
|
fi
|
|
elif [ "$exit_code" -eq 0 ]; then
|
|
overall_result="PASS"
|
|
fi
|
|
|
|
# exit code와 SUMMARY status가 어긋나면 안전하게 FAIL로 본다.
|
|
if [ "$exit_code" -ne 0 ] && [ "$overall_result" = "PASS" ]; then
|
|
overall_result="FAIL"
|
|
fi
|
|
|
|
build_rows_table() {
|
|
printf '| Profile | Axis | Requests | Throughput(rps) | p50(ms) | p95(ms) | p99(ms) | Timeout | NonceMis | TypeMis | FIFOViol | PendLeak | Gateway | Mem(MB) | 결과 |\n'
|
|
printf '|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---:|---|\n'
|
|
local line
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
ROW\|*) ;;
|
|
*) continue ;;
|
|
esac
|
|
IFS='|' read -r _tag profile axis requests throughput p50 p95 p99 timeouts noncemis typemis fifoviol pendleak gateway mem status <<<"$line"
|
|
printf '| %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s |\n' \
|
|
"$profile" "$axis" "$requests" "$throughput" "$p50" "$p95" "$p99" \
|
|
"$timeouts" "$noncemis" "$typemis" "$fifoviol" "$pendleak" "$gateway" "$mem" "$status"
|
|
done <"$stdout_file"
|
|
}
|
|
|
|
{
|
|
printf '%s\n' '---'
|
|
printf 'test_env: local\n'
|
|
printf 'record_type: stress-result\n'
|
|
printf 'test_profile: %s\n' "$record_profile"
|
|
printf 'created_at: %s\n' "$run_started_at"
|
|
printf 'overall_result: %s\n' "$overall_result"
|
|
printf '%s\n\n' '---'
|
|
printf '# %s local 결과 기록\n\n' "$record_profile"
|
|
printf '## 실행 정보\n\n'
|
|
printf -- '- 실행 일시: %s\n' "$run_started_at"
|
|
printf -- '- git ref: %s\n' "$git_ref"
|
|
printf -- '- 실행 명령: `%s`\n' "$run_cmd"
|
|
printf -- '- mode: %s\n' "$mode"
|
|
printf -- '- profiles: %s\n' "$display_profiles"
|
|
printf -- '- exit code: %s\n' "$exit_code"
|
|
printf -- '- stability violations: %s\n' "$stability_violations"
|
|
printf -- '- 전체 결과값: %s\n\n' "$overall_result"
|
|
printf '## 안정성 합격선\n\n'
|
|
printf -- '- timeout / nonce mismatch / response type mismatch / per-connection FIFO violation / pending-queue leak 모두 0이어야 PASS.\n'
|
|
printf -- '- 절대 throughput/latency는 환경 의존 baseline으로 저장하며 고정 합격선으로 쓰지 않는다.\n\n'
|
|
printf '## 측정 결과\n\n'
|
|
build_rows_table
|
|
printf '\n## git status 요약\n\n'
|
|
printf '```text\n'
|
|
git -C "$repo_root" status --short 2>/dev/null || true
|
|
printf '```\n\n'
|
|
printf '## 원본 출력\n\n'
|
|
printf '```text\n'
|
|
cat "$stdout_file"
|
|
printf '```\n'
|
|
if [ -s "$stderr_file" ]; then
|
|
printf '\n## 진행 로그(stderr) tail\n\n'
|
|
printf '```text\n'
|
|
tail -n 40 "$stderr_file"
|
|
printf '```\n'
|
|
fi
|
|
} >"$record_file"
|
|
|
|
cat "$stdout_file"
|
|
printf '\n전체 결과값: %s\n' "$overall_result"
|
|
printf '결과 기록 파일: `%s`\n' "${record_file#$repo_root/}"
|
|
|
|
exit "$exit_code"
|