toki
d2cee0ab9b
- typescript: communicator nonce handling and WS client improvements - typescript: performance benchmark and test updates - agent-ops: update test matrix scripts for performance/stress - agent-roadmap: update milestone documentation - agent-task: add kotlin tcp parallel optimization task
424 lines
14 KiB
Bash
Executable file
424 lines
14 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -u
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: run_performance.sh [--quick|--full] [--no-cross]
|
|
[--baseline <record.md>] [--fail-on-regression]
|
|
[--throughput-drop-pct N] [--p99-worse-pct N]
|
|
|
|
Runs the Proto Socket performance validation standard separately from the
|
|
compatibility matrix. The wrapper executes run_stress.sh components, merges the
|
|
measured rows into one local record, and compares them with a prior performance
|
|
record when a baseline is supplied.
|
|
|
|
Defaults:
|
|
--quick
|
|
throughput regression warning threshold: 20%
|
|
p99 latency regression warning threshold: 25%
|
|
|
|
Recommended:
|
|
Quick performance signal:
|
|
bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_performance.sh --quick
|
|
|
|
Full local baseline candidate:
|
|
bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_performance.sh --full
|
|
|
|
Regression comparison:
|
|
bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_performance.sh --quick --baseline agent-test/runs/<baseline>.md
|
|
USAGE
|
|
}
|
|
|
|
mode="quick"
|
|
run_cross=1
|
|
baseline_file=""
|
|
fail_on_regression=0
|
|
throughput_drop_pct="20"
|
|
p99_worse_pct="25"
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--quick) mode="quick" ;;
|
|
--full) mode="full" ;;
|
|
--no-cross) run_cross=0 ;;
|
|
--baseline=*) baseline_file="${1#--baseline=}" ;;
|
|
--baseline) shift; baseline_file="${1:-}" ;;
|
|
--fail-on-regression) fail_on_regression=1 ;;
|
|
--throughput-drop-pct=*) throughput_drop_pct="${1#--throughput-drop-pct=}" ;;
|
|
--throughput-drop-pct) shift; throughput_drop_pct="${1:-}" ;;
|
|
--p99-worse-pct=*) p99_worse_pct="${1#--p99-worse-pct=}" ;;
|
|
--p99-worse-pct) shift; p99_worse_pct="${1:-}" ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
case "$mode" in
|
|
quick|full) ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
|
|
case "$throughput_drop_pct" in
|
|
''|*[!0-9.]*)
|
|
printf 'invalid --throughput-drop-pct: %s\n' "$throughput_drop_pct" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
case "$p99_worse_pct" in
|
|
''|*[!0-9.]*)
|
|
printf 'invalid --p99-worse-pct: %s\n' "$p99_worse_pct" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
repo_root="$(cd "$script_dir/../../../../.." && pwd)"
|
|
stress_script="$script_dir/run_stress.sh"
|
|
run_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
run_record_stamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
record_profile="proto-socket-performance-${mode}"
|
|
record_dir="$repo_root/agent-test/runs"
|
|
record_file="$record_dir/${run_record_stamp}-${record_profile}.md"
|
|
log_dir="$(mktemp -d "${TMPDIR:-/tmp}/proto-socket-performance.XXXXXX")"
|
|
git_ref="$(git -C "$repo_root" rev-parse --short HEAD 2>/dev/null || echo "unknown")"
|
|
|
|
mkdir -p "$record_dir"
|
|
|
|
if [ -n "$baseline_file" ]; then
|
|
case "$baseline_file" in
|
|
/*) ;;
|
|
*) baseline_file="$repo_root/$baseline_file" ;;
|
|
esac
|
|
if [ ! -f "$baseline_file" ]; then
|
|
printf 'baseline file not found: %s\n' "$baseline_file" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
run_cmd="bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_performance.sh --$mode"
|
|
[ "$run_cross" -eq 0 ] && run_cmd="$run_cmd --no-cross"
|
|
[ -n "$baseline_file" ] && run_cmd="$run_cmd --baseline ${baseline_file#$repo_root/}"
|
|
[ "$fail_on_regression" -eq 1 ] && run_cmd="$run_cmd --fail-on-regression"
|
|
run_cmd="$run_cmd --throughput-drop-pct $throughput_drop_pct --p99-worse-pct $p99_worse_pct"
|
|
|
|
component_names=()
|
|
component_cmds=()
|
|
component_logs=()
|
|
component_records=()
|
|
component_results=()
|
|
component_exits=()
|
|
|
|
add_component() {
|
|
component_names+=("$1")
|
|
component_cmds+=("$2")
|
|
}
|
|
|
|
same_profiles="roundtrip,burst,sustained,parallel,payload"
|
|
all_langs="dart,go,kotlin,python,typescript"
|
|
all_transports="tcp,ws"
|
|
|
|
add_component "same-language" "bash $(printf '%q' "$stress_script") --$mode --lang $all_langs --transport $all_transports --profile $same_profiles"
|
|
if [ "$run_cross" -eq 1 ]; then
|
|
add_component "cross-language" "bash $(printf '%q' "$stress_script") --$mode --lang $all_langs --transport $all_transports --profile cross"
|
|
fi
|
|
add_component "typescript-gateway" "bash $(printf '%q' "$stress_script") --$mode --lang typescript --transport $all_transports --profile gateway"
|
|
|
|
component_result_from_exit() {
|
|
local exit_code="$1" parsed="$2"
|
|
if [ -n "$parsed" ]; then
|
|
printf '%s' "$parsed"
|
|
return
|
|
fi
|
|
case "$exit_code" in
|
|
0) printf 'PASS' ;;
|
|
3) printf 'BLOCKED' ;;
|
|
4) printf 'INCOMPLETE' ;;
|
|
*) printf 'FAIL' ;;
|
|
esac
|
|
}
|
|
|
|
run_component() {
|
|
local index="$1"
|
|
local name="${component_names[$index]}"
|
|
local cmd="${component_cmds[$index]}"
|
|
local log_file="$log_dir/${name}.log"
|
|
local exit_code parsed_result record_path
|
|
|
|
printf 'RUN performance %s: %s\n' "$name" "$cmd" >&2
|
|
(
|
|
cd "$repo_root" &&
|
|
bash -c "$cmd"
|
|
) >"$log_file" 2>&1
|
|
exit_code=$?
|
|
|
|
tail -n 20 "$log_file" >&2 || true
|
|
|
|
parsed_result="$(sed -n 's/^- 전체 결과값: //p; s/^전체 결과값: //p' "$log_file" | tail -n 1)"
|
|
record_path="$(sed -n 's/^결과 기록 파일: `\(.*\)`/\1/p' "$log_file" | tail -n 1)"
|
|
if [ -n "$record_path" ]; then
|
|
case "$record_path" in
|
|
/*) ;;
|
|
*) record_path="$repo_root/$record_path" ;;
|
|
esac
|
|
fi
|
|
|
|
component_logs[$index]="$log_file"
|
|
component_records[$index]="$record_path"
|
|
component_results[$index]="$(component_result_from_exit "$exit_code" "$parsed_result")"
|
|
component_exits[$index]="$exit_code"
|
|
}
|
|
|
|
extract_measurement_rows() {
|
|
local file="$1"
|
|
[ -f "$file" ] || return 0
|
|
awk '
|
|
/^## 측정 결과/ { in_rows=1; next }
|
|
in_rows && /^## / { in_rows=0 }
|
|
in_rows && /^\| / {
|
|
if ($0 ~ /^\|---/) next
|
|
if ($0 ~ /^\| Profile /) next
|
|
print
|
|
}
|
|
' "$file"
|
|
}
|
|
|
|
build_component_summary() {
|
|
local i
|
|
printf '| 구성 | 결과 | exit code | 결과 기록 | 로그 |\n'
|
|
printf '|---|---|---:|---|---|\n'
|
|
for i in "${!component_names[@]}"; do
|
|
local record="${component_records[$i]:-}"
|
|
local log="${component_logs[$i]:-}"
|
|
[ -n "$record" ] && record="${record#$repo_root/}"
|
|
[ -n "$log" ] && log="$log"
|
|
printf '| %s | %s | %s | %s | %s |\n' \
|
|
"${component_names[$i]}" "${component_results[$i]:--}" "${component_exits[$i]:--}" \
|
|
"${record:-미기록}" "$log"
|
|
done
|
|
}
|
|
|
|
trim_awk='
|
|
function trim(s) {
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", s)
|
|
return s
|
|
}
|
|
'
|
|
|
|
compare_rows() {
|
|
local baseline_rows="$1" current_rows="$2" out_file="$3" status_file="$4"
|
|
awk -F'|' \
|
|
-v throughput_limit="$throughput_drop_pct" \
|
|
-v p99_limit="$p99_worse_pct" \
|
|
-v status_file="$status_file" \
|
|
"$trim_awk"'
|
|
function row_key() {
|
|
return profile "|" axis "|" language "|" transport "|" payload "|" clients "|" gateway
|
|
}
|
|
function parse_row() {
|
|
profile=trim($2)
|
|
axis=trim($3)
|
|
language=trim($4)
|
|
transport=trim($5)
|
|
payload=trim($6)
|
|
clients=trim($7)
|
|
request_count=trim($8)
|
|
throughput=trim($9) + 0
|
|
p99=trim($12) + 0
|
|
gateway=trim($20)
|
|
result=trim($22)
|
|
}
|
|
BEGIN {
|
|
warning_count=0
|
|
compared_count=0
|
|
missing_count=0
|
|
printf "| Key | Baseline throughput | Current throughput | Drop %% | Baseline p99 | Current p99 | Worse %% | 결과 |\n"
|
|
printf "|---|---:|---:|---:|---:|---:|---:|---|\n"
|
|
}
|
|
FNR == NR {
|
|
parse_row()
|
|
key=row_key()
|
|
base_throughput[key]=throughput
|
|
base_p99[key]=p99
|
|
next
|
|
}
|
|
{
|
|
parse_row()
|
|
key=row_key()
|
|
label=sprintf("%s / %s / %s / %s / %sB / clients=%s / requests=%s / gateway=%s", profile, axis, language, transport, payload, clients, request_count, gateway)
|
|
if (!(key in base_throughput)) {
|
|
missing_count++
|
|
printf "| %s | - | %.3f | - | - | %.3f | - | SKIPPED(no baseline) |\n", label, throughput, p99
|
|
next
|
|
}
|
|
|
|
compared_count++
|
|
drop=0
|
|
worse=0
|
|
row_status="PASS"
|
|
if (base_throughput[key] > 0) {
|
|
drop=((base_throughput[key] - throughput) / base_throughput[key]) * 100
|
|
}
|
|
if (base_p99[key] > 0) {
|
|
worse=((p99 - base_p99[key]) / base_p99[key]) * 100
|
|
}
|
|
if (drop >= throughput_limit || worse >= p99_limit) {
|
|
row_status="WARN"
|
|
warning_count++
|
|
}
|
|
printf "| %s | %.3f | %.3f | %.1f | %.3f | %.3f | %.1f | %s |\n", \
|
|
label, base_throughput[key], throughput, drop, base_p99[key], p99, worse, row_status
|
|
}
|
|
END {
|
|
if (warning_count > 0) {
|
|
print "WARN" > status_file
|
|
} else {
|
|
print "PASS" > status_file
|
|
}
|
|
printf "%d\n", compared_count > status_file ".compared"
|
|
printf "%d\n", missing_count > status_file ".missing"
|
|
printf "%d\n", warning_count > status_file ".warnings"
|
|
}
|
|
' "$baseline_rows" "$current_rows" >"$out_file"
|
|
}
|
|
|
|
for i in "${!component_names[@]}"; do
|
|
run_component "$i"
|
|
done
|
|
|
|
overall_result="PASS"
|
|
for result in "${component_results[@]}"; do
|
|
case "$result" in
|
|
FAIL) overall_result="FAIL" ;;
|
|
esac
|
|
done
|
|
if [ "$overall_result" = "PASS" ]; then
|
|
for result in "${component_results[@]}"; do
|
|
case "$result" in
|
|
INCOMPLETE) overall_result="INCOMPLETE" ;;
|
|
esac
|
|
done
|
|
fi
|
|
if [ "$overall_result" = "PASS" ]; then
|
|
for result in "${component_results[@]}"; do
|
|
case "$result" in
|
|
BLOCKED) overall_result="BLOCKED" ;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
current_rows_file="$log_dir/current.rows"
|
|
: >"$current_rows_file"
|
|
for record in "${component_records[@]}"; do
|
|
extract_measurement_rows "$record" >>"$current_rows_file"
|
|
done
|
|
|
|
regression_result="SKIPPED"
|
|
comparison_file="$log_dir/regression-comparison.md"
|
|
compared_count=0
|
|
missing_count=0
|
|
warning_count=0
|
|
if [ -n "$baseline_file" ]; then
|
|
baseline_rows_file="$log_dir/baseline.rows"
|
|
extract_measurement_rows "$baseline_file" >"$baseline_rows_file"
|
|
status_file="$log_dir/regression.status"
|
|
if compare_rows "$baseline_rows_file" "$current_rows_file" "$comparison_file" "$status_file"; then
|
|
regression_result="$(cat "$status_file" 2>/dev/null || echo "PASS")"
|
|
compared_count="$(cat "$status_file.compared" 2>/dev/null || echo "0")"
|
|
missing_count="$(cat "$status_file.missing" 2>/dev/null || echo "0")"
|
|
warning_count="$(cat "$status_file.warnings" 2>/dev/null || echo "0")"
|
|
if [ "$overall_result" = "PASS" ] && [ "$regression_result" = "WARN" ]; then
|
|
overall_result="WARN"
|
|
fi
|
|
else
|
|
regression_result="FAIL"
|
|
overall_result="FAIL"
|
|
printf '| 오류 | baseline 비교 파서 실패 | - | - | - | - | - | FAIL |\n' >"$comparison_file"
|
|
fi
|
|
fi
|
|
|
|
{
|
|
printf '%s\n' '---'
|
|
printf 'test_env: local\n'
|
|
printf 'record_type: performance-result\n'
|
|
printf 'test_profile: %s\n' "$record_profile"
|
|
printf 'created_at: %s\n' "$run_started_at"
|
|
printf 'overall_result: %s\n' "$overall_result"
|
|
printf 'regression_result: %s\n' "$regression_result"
|
|
printf '%s\n\n' '---'
|
|
printf '# %s local 결과 기록\n\n' "$record_profile"
|
|
printf '## 실행 정보\n\n'
|
|
printf -- '- 실행 일시: %s\n' "$run_started_at"
|
|
printf -- '- git ref: %s\n' "$git_ref"
|
|
printf -- '- 실행 명령: `%s`\n' "$run_cmd"
|
|
printf -- '- mode: %s\n' "$mode"
|
|
printf -- '- 로그 디렉터리: `%s`\n' "$log_dir"
|
|
if [ -n "$baseline_file" ]; then
|
|
printf -- '- baseline: `%s`\n' "${baseline_file#$repo_root/}"
|
|
else
|
|
printf -- '- baseline: 없음\n'
|
|
fi
|
|
printf -- '- throughput regression threshold: %s%% drop\n' "$throughput_drop_pct"
|
|
printf -- '- p99 regression threshold: %s%% worse\n' "$p99_worse_pct"
|
|
printf -- '- 전체 결과값: %s\n' "$overall_result"
|
|
printf -- '- regression 결과값: %s\n\n' "$regression_result"
|
|
|
|
printf '## 성능 판정 기준\n\n'
|
|
printf -- '- stability hard gate: timeout, nonce mismatch, response type mismatch, per-connection FIFO violation, pending leak, queue/gateway backlog leak가 모두 0이어야 한다.\n'
|
|
printf -- '- performance regression gate: baseline이 주어지면 같은 row key에서 throughput %s%% 이상 하락 또는 p99 latency %s%% 이상 악화를 WARN으로 기록한다.\n' "$throughput_drop_pct" "$p99_worse_pct"
|
|
printf -- '- same-language stress row key에는 harness topology가 axis에 포함될 수 있으며, baseline 비교는 같은 topology 조건끼리 해석한다.\n'
|
|
printf -- '- 절대 throughput/latency는 local 환경 기준값이며, baseline 비교는 같은 host/runtime/profile에서 실행한 결과끼리 수행한다.\n'
|
|
printf -- '- `--fail-on-regression`을 지정하면 regression WARN도 non-zero exit로 처리한다.\n\n'
|
|
|
|
printf '## 구성요소 요약\n\n'
|
|
build_component_summary
|
|
|
|
printf '\n## 측정 결과\n\n'
|
|
printf '| Profile | Axis | Language | Transport | Payload(bytes) | Clients | Requests | Throughput(rps) | p50(ms) | p95(ms) | p99(ms) | Timeout | NonceMis | TypeMis | FIFOViol | PendLeak | QueueBacklog | GatewayBacklog | Gateway | Mem(MB) | 결과 |\n'
|
|
printf '|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---:|---|\n'
|
|
cat "$current_rows_file"
|
|
|
|
printf '\n## Regression 비교\n\n'
|
|
if [ -n "$baseline_file" ]; then
|
|
printf -- '- baseline: `%s`\n' "${baseline_file#$repo_root/}"
|
|
printf -- '- compared rows: %s\n' "$compared_count"
|
|
printf -- '- missing baseline rows: %s\n' "$missing_count"
|
|
printf -- '- warning rows: %s\n\n' "$warning_count"
|
|
cat "$comparison_file"
|
|
else
|
|
printf -- '- SKIPPED: `--baseline <record.md>`가 지정되지 않았다.\n'
|
|
fi
|
|
|
|
printf '\n## git status 요약\n\n'
|
|
printf '```text\n'
|
|
git -C "$repo_root" status --short 2>/dev/null || true
|
|
printf '```\n\n'
|
|
|
|
printf '## 구성요소 로그 tail\n\n'
|
|
for i in "${!component_names[@]}"; do
|
|
printf '### %s\n\n' "${component_names[$i]}"
|
|
printf '```text\n'
|
|
tail -n 30 "${component_logs[$i]}" 2>/dev/null || true
|
|
printf '```\n\n'
|
|
done
|
|
} >"$record_file"
|
|
|
|
printf '\n=== performance 측정 결과 ===\n' >&2
|
|
build_component_summary >&2
|
|
printf '\n전체 결과값: %s\n' "$overall_result"
|
|
printf 'Regression 결과값: %s\n' "$regression_result"
|
|
printf '결과 기록 파일: `%s`\n' "${record_file#$repo_root/}"
|
|
|
|
case "$overall_result" in
|
|
PASS) exit 0 ;;
|
|
WARN)
|
|
if [ "$fail_on_regression" -eq 1 ]; then
|
|
exit 5
|
|
fi
|
|
exit 0
|
|
;;
|
|
BLOCKED) exit 3 ;;
|
|
INCOMPLETE) exit 4 ;;
|
|
*) exit 1 ;;
|
|
esac
|