635 lines
28 KiB
Bash
Executable file
635 lines
28 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: run_stress.sh [--quick|--full] [--lang a,b,...] [--transport tcp,ws] [--profile a,b,...]
|
|
|
|
Runs the Proto Socket same-language stress / benchmark harness for one or more language
|
|
implementations and writes a single merged local result record under agent-test/runs/.
|
|
|
|
각 언어 구현은 동일한 출력 계약(ROW|/SKIP|/SUMMARY|)으로 same-language TCP/WS baseline을 낸다.
|
|
이 orchestrator는 선택한 언어를 각각 subprocess로 실행하고 결과 row를 하나의 기록으로 병합한다.
|
|
|
|
Modes:
|
|
--quick small, fast baseline for smoke/regression (default)
|
|
--full Load baseline candidates including 100k burst, 5m/30m sustained,
|
|
and 128/512/1024 client candidates where supported
|
|
|
|
Languages (comma separated, default: all):
|
|
dart go kotlin python typescript
|
|
|
|
Transports (comma separated, default: tcp,ws):
|
|
tcp same-language length-prefixed TCP request/response/burst baseline
|
|
ws same-language WebSocket binary-frame baseline
|
|
|
|
Profiles (comma separated, default: per-language all):
|
|
roundtrip request-response latency at concurrency 1/16/64/256
|
|
burst single-connection fire-and-forget burst ordering / leak check
|
|
(quick: 200, full: 1k/10k/100k)
|
|
sustained time-boxed sustained request-response with peak memory
|
|
(quick: 2s, full: 30s/5m/30m candidates)
|
|
parallel multi-connection FIFO / nonce isolation
|
|
(quick: 4 clients, full: 16/128/512/1024 client candidates)
|
|
slow-mix mixed fast/slow handler load, same-connection delay and cross-connection isolation
|
|
(quick: 4 clients, full: 16 clients)
|
|
payload payload size matrix request-response (quick: 1KB/64KB, full: 1KB/64KB/1MB)
|
|
with per-payload latency, throughput, peak memory, stability counters
|
|
cross cross-language request-response correctness / stability adapter
|
|
gateway (TypeScript only) worker_threads gateway frame-ingest/transport baseline
|
|
|
|
언어/transport별로 구현이 준비되지 않았거나 toolchain이 없으면 PASS로 기록하지 않고 BLOCKED 또는
|
|
SKIPPED(사유 포함)로 남긴다.
|
|
|
|
Stability pass criteria (hard fail): timeout / nonce mismatch / response type mismatch /
|
|
per-connection FIFO violation / pending-queue leak all 0. 절대 throughput/latency는 환경 의존
|
|
baseline으로 저장하며 고정 합격선으로 쓰지 않는다. TypeScript TCP full parallel 1024-client row는
|
|
single-process event-loop artifact를 피하기 위해 split-process server topology로 측정하고 row axis에
|
|
topology를 기록한다.
|
|
USAGE
|
|
}
|
|
|
|
mode="quick"
|
|
profiles=""
|
|
langs=""
|
|
transports=""
|
|
|
|
ALL_LANGS="dart go kotlin python typescript"
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--quick) mode="quick" ;;
|
|
--full) mode="full" ;;
|
|
--mode=*) mode="${1#--mode=}" ;;
|
|
--profile=*) profiles="${1#--profile=}" ;;
|
|
--profiles=*) profiles="${1#--profiles=}" ;;
|
|
--profile|--profiles) shift; profiles="${1:-}" ;;
|
|
--lang=*) langs="${1#--lang=}" ;;
|
|
--langs=*) langs="${1#--langs=}" ;;
|
|
--lang|--langs) shift; langs="${1:-}" ;;
|
|
--transport=*) transports="${1#--transport=}" ;;
|
|
--transports=*) transports="${1#--transports=}" ;;
|
|
--transport|--transports) shift; transports="${1:-}" ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
case "$mode" in
|
|
quick|full) ;;
|
|
*) usage >&2; exit 2 ;;
|
|
esac
|
|
|
|
# 콤마/공백 구분을 공백 토큰으로 정규화한다.
|
|
normalize_list() { printf '%s' "$1" | tr ',' ' ' | tr -s ' '; }
|
|
|
|
selected_langs="$(normalize_list "${langs:-$ALL_LANGS}")"
|
|
selected_transports="$(normalize_list "${transports:-tcp ws}")"
|
|
|
|
# 미지원 언어 토큰은 즉시 거른다.
|
|
valid_langs=""
|
|
for lang in $selected_langs; do
|
|
case " $ALL_LANGS " in
|
|
*" $lang "*) valid_langs="$valid_langs $lang" ;;
|
|
*) printf 'unknown language: %s (valid: %s)\n' "$lang" "$ALL_LANGS" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
selected_langs="$(printf '%s' "$valid_langs" | tr -s ' ' | sed 's/^ //')"
|
|
|
|
# 미지원 transport 토큰도 언어별 harness에 넘기기 전에 즉시 거른다.
|
|
valid_transports=""
|
|
for transport in $selected_transports; do
|
|
case "$transport" in
|
|
tcp|ws) valid_transports="$valid_transports $transport" ;;
|
|
*) printf 'unknown transport: %s (valid: tcp ws)\n' "$transport" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
selected_transports="$(printf '%s' "$valid_transports" | tr -s ' ' | sed 's/^ //')"
|
|
[ -n "$selected_transports" ] || { printf 'no transport selected (valid: tcp ws)\n' >&2; usage >&2; exit 2; }
|
|
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
repo_root="$(cd "$script_dir/../../../../.." && pwd)"
|
|
|
|
run_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
run_record_stamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
record_profile="proto-socket-stress-${mode}"
|
|
git_ref="$(git -C "$repo_root" rev-parse --short HEAD 2>/dev/null || echo "unknown")"
|
|
|
|
record_dir="$repo_root/agent-test/runs"
|
|
mkdir -p "$record_dir"
|
|
record_file="$record_dir/${run_record_stamp}-${record_profile}.md"
|
|
|
|
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/proto-socket-stress.XXXXXX")"
|
|
cleanup() { rm -rf "$work_dir"; }
|
|
trap cleanup EXIT
|
|
|
|
transports_csv="$(printf '%s' "$selected_transports" | tr ' ' ',')"
|
|
profiles_csv="$(printf '%s' "$(normalize_list "$profiles")" | tr ' ' ',')"
|
|
|
|
profile_selected() {
|
|
local needle="$1"
|
|
local selected
|
|
selected="$(normalize_list "$profiles")"
|
|
[ -z "$selected" ] && return 1
|
|
case " $selected " in
|
|
*" $needle "*) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# 사용자 재현용 실행 명령 문자열.
|
|
run_cmd="bash agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_stress.sh --$mode"
|
|
run_cmd="$run_cmd --lang $(printf '%s' "$selected_langs" | tr ' ' ',')"
|
|
run_cmd="$run_cmd --transport $transports_csv"
|
|
[ -n "$profiles_csv" ] && run_cmd="$run_cmd --profile $profiles_csv"
|
|
|
|
lang_available() {
|
|
case "$1" in
|
|
dart) command -v dart >/dev/null 2>&1 ;;
|
|
go) command -v go >/dev/null 2>&1 ;;
|
|
python) command -v python3 >/dev/null 2>&1 ;;
|
|
typescript) [ -x "$repo_root/typescript/node_modules/.bin/tsx" ] ;;
|
|
kotlin) [ -x "$repo_root/kotlin/gradlew" ] && command -v java >/dev/null 2>&1 ;;
|
|
esac
|
|
}
|
|
|
|
lang_block_reason() {
|
|
case "$1" in
|
|
dart) echo "dart SDK not found on PATH" ;;
|
|
go) echo "go toolchain not found on PATH" ;;
|
|
python) echo "python3 not found on PATH" ;;
|
|
typescript) echo "tsx runner not found (run 'npm install' in typescript/)" ;;
|
|
kotlin) echo "kotlin gradlew or JDK not available" ;;
|
|
esac
|
|
}
|
|
|
|
language_label() {
|
|
case "$1" in
|
|
dart) echo "Dart" ;;
|
|
go) echo "Go" ;;
|
|
kotlin) echo "Kotlin" ;;
|
|
python) echo "Python" ;;
|
|
typescript) echo "TypeScript" ;;
|
|
*) echo "$1" ;;
|
|
esac
|
|
}
|
|
|
|
cross_command_for() {
|
|
local server="$1" client="$2"
|
|
case "$server|$client" in
|
|
Go\|Dart) echo "$repo_root/go|go run ./crosstest/go_dart.go" ;;
|
|
Go\|Kotlin) echo "$repo_root/go|go run ./crosstest/go_kotlin.go" ;;
|
|
Go\|Python) echo "$repo_root/go|go run ./crosstest/go_python.go" ;;
|
|
Go\|TypeScript) echo "$repo_root/go|go run ./crosstest/go_typescript.go" ;;
|
|
Dart\|Go) echo "$repo_root/dart|dart run crosstest/dart_go.dart" ;;
|
|
Dart\|Kotlin) echo "$repo_root/dart|dart run crosstest/dart_kotlin.dart" ;;
|
|
Dart\|Python) echo "$repo_root/dart|dart run crosstest/dart_python.dart" ;;
|
|
Dart\|TypeScript) echo "$repo_root/dart|dart run crosstest/dart_typescript.dart" ;;
|
|
Kotlin\|Dart) echo "$repo_root/kotlin|./gradlew run -PmainClass=com.tokilabs.proto_socket.crosstest.KotlinDartKt" ;;
|
|
Kotlin\|Go) echo "$repo_root/kotlin|./gradlew run -PmainClass=com.tokilabs.proto_socket.crosstest.KotlinGoKt" ;;
|
|
Kotlin\|Python) echo "$repo_root/kotlin|./gradlew run -PmainClass=com.tokilabs.proto_socket.crosstest.KotlinPythonKt" ;;
|
|
Kotlin\|TypeScript) echo "$repo_root/kotlin|./gradlew run -PmainClass=com.tokilabs.proto_socket.crosstest.KotlinTypescriptKt" ;;
|
|
Python\|Dart) echo "$repo_root/python|python3 crosstest/python_dart.py" ;;
|
|
Python\|Go) echo "$repo_root/python|python3 crosstest/python_go.py" ;;
|
|
Python\|Kotlin) echo "$repo_root/python|python3 crosstest/python_kotlin.py" ;;
|
|
Python\|TypeScript) echo "$repo_root/python|python3 crosstest/python_typescript.py" ;;
|
|
TypeScript\|Dart) echo "$repo_root/typescript|./node_modules/.bin/tsx crosstest/typescript_dart.ts" ;;
|
|
TypeScript\|Go) echo "$repo_root/typescript|./node_modules/.bin/tsx crosstest/typescript_go.ts" ;;
|
|
TypeScript\|Kotlin) echo "$repo_root/typescript|./node_modules/.bin/tsx crosstest/typescript_kotlin.ts" ;;
|
|
TypeScript\|Python) echo "$repo_root/typescript|./node_modules/.bin/tsx crosstest/typescript_python.ts" ;;
|
|
esac
|
|
}
|
|
|
|
cross_transport_count() {
|
|
local _transport="$1" file="$2"
|
|
grep -c '^PASS scenario=' "$file" 2>/dev/null || true
|
|
}
|
|
|
|
cross_expected_count() {
|
|
case "$1" in
|
|
tcp) echo 5 ;;
|
|
ws) echo 4 ;;
|
|
*) echo 4 ;;
|
|
esac
|
|
}
|
|
|
|
run_cross_profile() {
|
|
local cross_record_profile="proto-socket-stress-${mode}-cross"
|
|
local cross_record_file="$record_dir/${run_record_stamp}-${cross_record_profile}.md"
|
|
local overall="PASS"
|
|
local rows_file="$work_dir/cross.rows"
|
|
local summary_file="$work_dir/cross.summary"
|
|
local failures_file="$work_dir/cross.failures"
|
|
: >"$rows_file"
|
|
: >"$summary_file"
|
|
: >"$failures_file"
|
|
|
|
local selected_server_slugs selected_client_slugs
|
|
selected_server_slugs="$selected_langs"
|
|
selected_client_slugs="$selected_langs"
|
|
|
|
for server_slug in $selected_server_slugs; do
|
|
local server
|
|
server="$(language_label "$server_slug")"
|
|
for client_slug in $selected_client_slugs; do
|
|
local client
|
|
client="$(language_label "$client_slug")"
|
|
[ "$server" = "$client" ] && continue
|
|
|
|
local cmd_spec dir cmd key
|
|
cmd_spec="$(cross_command_for "$server" "$client" || true)"
|
|
key="${server}_${client}"
|
|
if [ -z "$cmd_spec" ]; then
|
|
status="SKIPPED"
|
|
note="unsupported cross pair"
|
|
printf '| %s -> %s | %s | 0 | 16 | 0 | %s |\n' "$server" "$client" "$status" "$note" >>"$summary_file"
|
|
continue
|
|
fi
|
|
IFS='|' read -r dir cmd <<<"$cmd_spec"
|
|
|
|
if ! lang_available "$server_slug"; then
|
|
status="BLOCKED"
|
|
note="server runtime unavailable: $(lang_block_reason "$server_slug")"
|
|
overall="BLOCKED"
|
|
printf '| %s -> %s | %s | 0 | 16 | 0 | %s |\n' "$server" "$client" "$status" "$note" >>"$summary_file"
|
|
printf -- '- %s -> %s: `%s`; %s\n' "$server" "$client" "(cd ${dir#$repo_root/} && $cmd)" "$note" >>"$failures_file"
|
|
continue
|
|
fi
|
|
if ! lang_available "$client_slug"; then
|
|
status="BLOCKED"
|
|
note="client runtime unavailable: $(lang_block_reason "$client_slug")"
|
|
overall="BLOCKED"
|
|
printf '| %s -> %s | %s | 0 | 16 | 0 | %s |\n' "$server" "$client" "$status" "$note" >>"$summary_file"
|
|
printf -- '- %s -> %s: `%s`; %s\n' "$server" "$client" "(cd ${dir#$repo_root/} && $cmd)" "$note" >>"$failures_file"
|
|
continue
|
|
fi
|
|
|
|
for transport in $selected_transports; do
|
|
local expected_for_command
|
|
expected_for_command="$(cross_expected_count "$transport")"
|
|
for concurrency in 1 16 64 256; do
|
|
local log_file start_ns end_ns elapsed_ms pass_total fail_total status note throughput
|
|
log_file="$work_dir/cross_${key}_${transport}_c${concurrency}.log"
|
|
printf 'RUN cross stress server=%s client=%s transport=%s concurrency=%s command=%s\n' "$server" "$client" "$transport" "$concurrency" "$cmd" >&2
|
|
start_ns="$(date +%s%N)"
|
|
( cd "$dir" && PROTO_SOCKET_CROSSTEST_CONCURRENCY="$concurrency" PROTO_SOCKET_CROSSTEST_TRANSPORT="$transport" bash -c "$cmd" ) >"$log_file" 2>&1
|
|
exit_code=$?
|
|
end_ns="$(date +%s%N)"
|
|
elapsed_ms="$(( (end_ns - start_ns) / 1000000 ))"
|
|
[ "$elapsed_ms" -le 0 ] && elapsed_ms=1
|
|
tail -n 10 "$log_file" >&2 || true
|
|
|
|
pass_total="$(cross_transport_count "$transport" "$log_file")"
|
|
fail_total="$(grep -c '^FAIL' "$log_file" 2>/dev/null || true)"
|
|
if [ "$exit_code" -eq 0 ] && [ "$pass_total" -eq "$expected_for_command" ] && [ "$fail_total" -eq 0 ]; then
|
|
status="PASS"
|
|
note=""
|
|
else
|
|
status="FAIL"
|
|
note="transport=$transport concurrency=$concurrency exit=$exit_code pass=$pass_total expected=$expected_for_command fail=$fail_total"
|
|
overall="FAIL"
|
|
printf -- '- %s -> %s %s c%s: `%s`; %s\n' "$server" "$client" "$transport" "$concurrency" \
|
|
"(cd ${dir#$repo_root/} && PROTO_SOCKET_CROSSTEST_CONCURRENCY=$concurrency PROTO_SOCKET_CROSSTEST_TRANSPORT=$transport $cmd)" "$note" >>"$failures_file"
|
|
printf '```text\n' >>"$failures_file"
|
|
tail -n 20 "$log_file" >>"$failures_file" 2>/dev/null || true
|
|
printf '```\n' >>"$failures_file"
|
|
fi
|
|
printf '| %s -> %s %s c%s | %s | %s | %s | %s | %s |\n' "$server" "$client" "$transport" "$concurrency" "$status" "$pass_total" "$expected_for_command" "$fail_total" "$note" >>"$summary_file"
|
|
|
|
throughput="$(awk -v req="$concurrency" -v ms="$elapsed_ms" 'BEGIN { printf "%.2f", (req * 1000) / ms }')"
|
|
if [ "$status" = "PASS" ]; then
|
|
printf 'ROW|cross|request-response-c%s|%s->%s|%s|0|%s|%s|%s|%s|%s|%s|0|0|0|0|0|0|0|off|0|PASS\n' \
|
|
"$concurrency" "$server" "$client" "$transport" "$concurrency" "$concurrency" "$throughput" "$elapsed_ms" "$elapsed_ms" "$elapsed_ms" >>"$rows_file"
|
|
else
|
|
printf 'ROW|cross|request-response-c%s|%s->%s|%s|0|%s|%s|0|0|0|0|?|?|?|?|?|0|0|off|0|FAIL\n' \
|
|
"$concurrency" "$server" "$client" "$transport" "$concurrency" "$concurrency" >>"$rows_file"
|
|
fi
|
|
done
|
|
done
|
|
done
|
|
done
|
|
|
|
{
|
|
printf '%s\n' '---'
|
|
printf 'test_env: local\n'
|
|
printf 'record_type: stress-result\n'
|
|
printf 'test_profile: %s\n' "$cross_record_profile"
|
|
printf 'created_at: %s\n' "$run_started_at"
|
|
printf 'overall_result: %s\n' "$overall"
|
|
printf '%s\n\n' '---'
|
|
printf '# %s local 결과 기록\n\n' "$cross_record_profile"
|
|
printf '## 실행 정보\n\n'
|
|
printf -- '- 실행 일시: %s\n' "$run_started_at"
|
|
printf -- '- git ref: %s\n' "$git_ref"
|
|
printf -- '- 실행 명령: `%s`\n' "$run_cmd"
|
|
printf -- '- mode: %s\n' "$mode"
|
|
printf -- '- languages: %s\n' "$(printf '%s' "$selected_langs" | tr ' ' ',')"
|
|
printf -- '- transports: %s\n' "$transports_csv"
|
|
printf -- '- profiles: cross\n'
|
|
printf -- '- 전체 결과값: %s\n\n' "$overall"
|
|
printf '## 안정성 합격선\n\n'
|
|
printf -- '- timeout / nonce mismatch / response type mismatch / per-connection FIFO violation / pending-queue leak 모두 0이어야 PASS.\n'
|
|
printf -- '- 이 cross adapter는 기존 crosstest request-response 병렬 scenario를 공통 stress row schema로 기록한다.\n'
|
|
printf -- '- cross adapter는 transport/concurrency 조합마다 `PROTO_SOCKET_CROSSTEST_TRANSPORT`를 지정해 별도 command를 실행한다.\n'
|
|
printf -- '- crosstest command가 실패하거나 선택 transport의 기대 PASS scenario 수와 정확히 일치하지 않으면 해당 row는 FAIL이다.\n\n'
|
|
printf '## 크로스 언어 요약\n\n'
|
|
printf '| 방향 | 결과 | PASS scenarios | Expected | FAIL lines | 비고 |\n'
|
|
printf '|---|---|---:|---:|---:|---|\n'
|
|
cat "$summary_file"
|
|
printf '\n## 측정 결과\n\n'
|
|
printf '| Profile | Axis | Language | Transport | Payload(bytes) | Clients | Requests | Throughput(rps) | p50(ms) | p95(ms) | p99(ms) | Timeout | NonceMis | TypeMis | FIFOViol | PendLeak | QueueBacklog | GatewayBacklog | Gateway | Mem(MB) | 결과 |\n'
|
|
printf '|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---:|---|\n'
|
|
while IFS= read -r row_line; do
|
|
case "$row_line" in ROW\|*) ;; *) continue ;; esac
|
|
IFS='|' read -r _tag profile axis language transport payload clients requests throughput p50 p95 p99 timeouts noncemis typemis fifoviol pendleak queue_backlog gateway_backlog gateway mem status <<<"$row_line"
|
|
printf '| %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s |\n' \
|
|
"$profile" "$axis" "$language" "$transport" "$payload" "$clients" "$requests" "$throughput" "$p50" "$p95" "$p99" \
|
|
"$timeouts" "$noncemis" "$typemis" "$fifoviol" "$pendleak" "$queue_backlog" "$gateway_backlog" "$gateway" "$mem" "$status"
|
|
done <"$rows_file"
|
|
printf '\n## 실패/차단\n\n'
|
|
if [ -s "$failures_file" ]; then
|
|
cat "$failures_file"
|
|
else
|
|
printf -- '- 없음\n'
|
|
fi
|
|
printf '\n## git status 요약\n\n'
|
|
printf '```text\n'
|
|
git -C "$repo_root" status --short 2>/dev/null || true
|
|
printf '```\n\n'
|
|
printf '## 크로스 진행 로그 tail\n\n'
|
|
for log_file in "$work_dir"/cross_*.log; do
|
|
[ -f "$log_file" ] || continue
|
|
printf '### %s\n\n' "$(basename "$log_file" .log)"
|
|
printf '```text\n'
|
|
tail -n 20 "$log_file" 2>/dev/null || true
|
|
printf '```\n\n'
|
|
done
|
|
} >"$cross_record_file"
|
|
|
|
printf '\n=== cross stress 측정 결과 ===\n' >&2
|
|
cat "$summary_file" >&2
|
|
printf '\n전체 결과값: %s\n' "$overall"
|
|
printf '결과 기록 파일: `%s`\n' "${cross_record_file#$repo_root/}"
|
|
case "$overall" in
|
|
PASS) exit 0 ;;
|
|
BLOCKED) exit 3 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
}
|
|
|
|
# SKIP reason을 optional/required로 분류한다.
|
|
#
|
|
# optional skip은 그 언어의 same-language baseline 대상이 아닌 축이라 전체 PASS를 깨지 않는다
|
|
# (예: gateway는 TypeScript 전용, 미지원 transport). required skip은 기본 선택 범위에서 측정돼야 하는
|
|
# 축이 측정되지 못한 경우(예: optimize Epic으로 이연된 stability 병목)라, 포함되면 전체 결과를 PASS로
|
|
# 기록하지 않는다. 알 수 없는 reason은 안전하게 required로 본다. 반환: required=0, optional=1.
|
|
skip_is_required() {
|
|
case "$1" in
|
|
optional:*) return 1 ;;
|
|
*"not part of"*"same-language baseline"*) return 1 ;;
|
|
*"unsupported transport"*) return 1 ;;
|
|
*) return 0 ;;
|
|
esac
|
|
}
|
|
|
|
# 언어별 stress 진입점을 실행한다. stdout=ROW|/SKIP|/SUMMARY|, stderr=진행 로그.
|
|
run_lang() {
|
|
local lang="$1" outfile="$2" errfile="$3"
|
|
local margs="--mode=$mode" targs="--transport=$transports_csv"
|
|
local pargs=""
|
|
[ -n "$profiles_csv" ] && pargs="--profiles=$profiles_csv"
|
|
|
|
case "$lang" in
|
|
dart)
|
|
( cd "$repo_root/dart" && dart run bench/stress.dart "$margs" "$targs" ${pargs:+"$pargs"} ) >"$outfile" 2>"$errfile" ;;
|
|
go)
|
|
( cd "$repo_root/go" && go run ./bench/stress.go "$margs" "$targs" ${pargs:+"$pargs"} ) >"$outfile" 2>"$errfile" ;;
|
|
python)
|
|
( cd "$repo_root/python" && python3 bench/stress.py "$margs" "$targs" ${pargs:+"$pargs"} ) >"$outfile" 2>"$errfile" ;;
|
|
typescript)
|
|
( cd "$repo_root/typescript" && ./node_modules/.bin/tsx bench/stress.ts "$margs" "$targs" ${pargs:+"$pargs"} ) >"$outfile" 2>"$errfile" ;;
|
|
kotlin)
|
|
local kargs="$margs $targs"
|
|
[ -n "$pargs" ] && kargs="$kargs $pargs"
|
|
( cd "$repo_root/kotlin" && ./gradlew --offline -q run -PmainClass=com.tokilabs.proto_socket.crosstest.StressKt --args="$kargs" ) >"$outfile" 2>"$errfile" ;;
|
|
esac
|
|
}
|
|
|
|
if profile_selected "cross"; then
|
|
run_cross_profile
|
|
fi
|
|
|
|
declare -A lang_result
|
|
declare -A lang_rows
|
|
declare -A lang_skips
|
|
declare -A lang_req_skips
|
|
declare -A lang_violations
|
|
declare -A lang_reason
|
|
|
|
any_fail=0
|
|
any_blocked=0
|
|
any_incomplete=0
|
|
|
|
for lang in $selected_langs; do
|
|
outfile="$work_dir/${lang}.out"
|
|
errfile="$work_dir/${lang}.err"
|
|
: >"$outfile"
|
|
: >"$errfile"
|
|
lang_req_skips[$lang]=0
|
|
|
|
if ! lang_available "$lang"; then
|
|
lang_result[$lang]="BLOCKED"
|
|
lang_reason[$lang]="$(lang_block_reason "$lang")"
|
|
lang_rows[$lang]=0
|
|
lang_skips[$lang]=0
|
|
lang_violations[$lang]="?"
|
|
printf 'BLOCKED %s: %s\n' "$lang" "${lang_reason[$lang]}" >&2
|
|
any_blocked=1
|
|
continue
|
|
fi
|
|
|
|
printf 'RUN stress lang=%s mode=%s transports=%s profiles=%s\n' \
|
|
"$lang" "$mode" "$transports_csv" "${profiles_csv:-all}" >&2
|
|
run_lang "$lang" "$outfile" "$errfile"
|
|
exit_code=$?
|
|
|
|
# 진행 로그를 흘려 사용자가 실행 흐름을 볼 수 있게 한다.
|
|
cat "$errfile" >&2
|
|
|
|
summary_line="$(grep '^SUMMARY|' "$outfile" | tail -n 1 || true)"
|
|
rows_count="$(grep -c '^ROW|' "$outfile" 2>/dev/null || true)"
|
|
skips_count="$(grep -c '^SKIP|' "$outfile" 2>/dev/null || true)"
|
|
rows_count="${rows_count:-0}"
|
|
skips_count="${skips_count:-0}"
|
|
lang_rows[$lang]="$rows_count"
|
|
lang_skips[$lang]="$skips_count"
|
|
|
|
# SKIP을 optional/required로 분류한다. required skip이 하나라도 있으면 그 언어의 baseline은
|
|
# 측정 범위 안에서 완결되지 않은 것이라 PASS로 보지 않는다.
|
|
req_skips=0
|
|
first_req_reason=""
|
|
while IFS= read -r sline; do
|
|
case "$sline" in SKIP\|*) ;; *) continue ;; esac
|
|
sreason="${sline##*|}"
|
|
if skip_is_required "$sreason"; then
|
|
req_skips=$((req_skips + 1))
|
|
[ -z "$first_req_reason" ] && first_req_reason="$sreason"
|
|
fi
|
|
done <"$outfile"
|
|
lang_req_skips[$lang]="$req_skips"
|
|
|
|
status="FAIL"
|
|
violations="?"
|
|
if [ -n "$summary_line" ]; then
|
|
status_field="$(printf '%s' "$summary_line" | tr '|' '\n' | sed -n 's/^status=//p' | head -n 1)"
|
|
violations_field="$(printf '%s' "$summary_line" | tr '|' '\n' | sed -n 's/^stability_violations=//p' | head -n 1)"
|
|
[ -n "$status_field" ] && status="$status_field"
|
|
[ -n "$violations_field" ] && violations="$violations_field"
|
|
elif [ "$exit_code" -eq 0 ] && [ "$rows_count" -gt 0 ]; then
|
|
status="PASS"
|
|
fi
|
|
# exit code와 SUMMARY status가 어긋나면 안전하게 FAIL로 본다.
|
|
if [ "$exit_code" -ne 0 ] && [ "$status" = "PASS" ]; then
|
|
status="FAIL"
|
|
fi
|
|
if [ -z "$summary_line" ] && [ "$rows_count" -eq 0 ]; then
|
|
status="BLOCKED"
|
|
lang_reason[$lang]="harness produced no result (exit=$exit_code); see stderr tail"
|
|
fi
|
|
# stability는 통과했지만 required 축이 SKIP된 경우 baseline 미완결로 본다.
|
|
if [ "$status" = "PASS" ] && [ "$req_skips" -gt 0 ]; then
|
|
status="INCOMPLETE"
|
|
lang_reason[$lang]="required SKIP=$req_skips (${first_req_reason})"
|
|
fi
|
|
|
|
lang_result[$lang]="$status"
|
|
lang_violations[$lang]="$violations"
|
|
|
|
case "$status" in
|
|
FAIL) any_fail=1 ;;
|
|
BLOCKED) any_blocked=1 ;;
|
|
INCOMPLETE) any_incomplete=1 ;;
|
|
esac
|
|
printf 'DONE %s: %s (rows=%s skips=%s required_skips=%s violations=%s)\n' \
|
|
"$lang" "$status" "$rows_count" "$skips_count" "$req_skips" "$violations" >&2
|
|
done
|
|
|
|
# 전체 결과: 실제 stability 실패가 가장 우선, 그다음 required 축 미충족(INCOMPLETE),
|
|
# 그다음 toolchain 미가용(BLOCKED), 모두 없으면 PASS.
|
|
if [ "$any_fail" -eq 1 ]; then
|
|
overall_result="FAIL"
|
|
elif [ "$any_incomplete" -eq 1 ]; then
|
|
overall_result="INCOMPLETE"
|
|
elif [ "$any_blocked" -eq 1 ]; then
|
|
overall_result="BLOCKED"
|
|
else
|
|
overall_result="PASS"
|
|
fi
|
|
|
|
build_rows_table() {
|
|
printf '| Profile | Axis | Language | Transport | Payload(bytes) | Clients | Requests | Throughput(rps) | p50(ms) | p95(ms) | p99(ms) | Timeout | NonceMis | TypeMis | FIFOViol | PendLeak | QueueBacklog | GatewayBacklog | Gateway | Mem(MB) | 결과 |\n'
|
|
printf '|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---:|---|\n'
|
|
local line lang
|
|
for lang in $selected_langs; do
|
|
[ -f "$work_dir/${lang}.out" ] || continue
|
|
while IFS= read -r line; do
|
|
case "$line" in ROW\|*) ;; *) continue ;; esac
|
|
IFS='|' read -r _tag profile axis language transport payload clients requests throughput p50 p95 p99 timeouts noncemis typemis fifoviol pendleak queue_backlog gateway_backlog gateway mem status <<<"$line"
|
|
printf '| %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s |\n' \
|
|
"$profile" "$axis" "$language" "$transport" "$payload" "$clients" "$requests" "$throughput" "$p50" "$p95" "$p99" \
|
|
"$timeouts" "$noncemis" "$typemis" "$fifoviol" "$pendleak" "$queue_backlog" "$gateway_backlog" "$gateway" "$mem" "$status"
|
|
done <"$work_dir/${lang}.out"
|
|
done
|
|
}
|
|
|
|
build_skips_table() {
|
|
local has_skip=0 line lang
|
|
for lang in $selected_langs; do
|
|
[ -f "$work_dir/${lang}.out" ] || continue
|
|
if grep -q '^SKIP|' "$work_dir/${lang}.out"; then has_skip=1; fi
|
|
done
|
|
[ "$has_skip" -eq 0 ] && { printf -- '- 없음\n'; return; }
|
|
printf '분류: `required`=기본 선택 범위에서 측정돼야 하나 미충족(전체 PASS 불가), `optional`=해당 언어 baseline 대상 외 축.\n\n'
|
|
printf '| 분류 | Profile | Axis | Language | Transport | 사유 |\n'
|
|
printf '|---|---|---|---|---|---|\n'
|
|
for lang in $selected_langs; do
|
|
[ -f "$work_dir/${lang}.out" ] || continue
|
|
while IFS= read -r line; do
|
|
case "$line" in SKIP\|*) ;; *) continue ;; esac
|
|
IFS='|' read -r _tag profile axis language transport reason <<<"$line"
|
|
local kind="optional"
|
|
skip_is_required "$reason" && kind="required"
|
|
printf '| %s | %s | %s | %s | %s | %s |\n' "$kind" "$profile" "$axis" "$language" "$transport" "$reason"
|
|
done <"$work_dir/${lang}.out"
|
|
done
|
|
}
|
|
|
|
build_lang_summary_table() {
|
|
printf '| 언어 | 결과 | ROW | SKIP(전체) | SKIP(required) | stability_violations | 비고 |\n'
|
|
printf '|---|---|---:|---:|---:|---:|---|\n'
|
|
local lang
|
|
for lang in $selected_langs; do
|
|
printf '| %s | %s | %s | %s | %s | %s | %s |\n' \
|
|
"$lang" "${lang_result[$lang]:--}" "${lang_rows[$lang]:-0}" "${lang_skips[$lang]:-0}" \
|
|
"${lang_req_skips[$lang]:-0}" "${lang_violations[$lang]:-?}" "${lang_reason[$lang]:-}"
|
|
done
|
|
}
|
|
|
|
{
|
|
printf '%s\n' '---'
|
|
printf 'test_env: local\n'
|
|
printf 'record_type: stress-result\n'
|
|
printf 'test_profile: %s\n' "$record_profile"
|
|
printf 'created_at: %s\n' "$run_started_at"
|
|
printf 'overall_result: %s\n' "$overall_result"
|
|
printf '%s\n\n' '---'
|
|
printf '# %s local 결과 기록\n\n' "$record_profile"
|
|
printf '## 실행 정보\n\n'
|
|
printf -- '- 실행 일시: %s\n' "$run_started_at"
|
|
printf -- '- git ref: %s\n' "$git_ref"
|
|
printf -- '- 실행 명령: `%s`\n' "$run_cmd"
|
|
printf -- '- mode: %s\n' "$mode"
|
|
printf -- '- languages: %s\n' "$(printf '%s' "$selected_langs" | tr ' ' ',')"
|
|
printf -- '- transports: %s\n' "$transports_csv"
|
|
printf -- '- profiles: %s\n' "${profiles_csv:-all}"
|
|
printf -- '- 전체 결과값: %s\n\n' "$overall_result"
|
|
printf '## 안정성 합격선\n\n'
|
|
printf -- '- timeout / nonce mismatch / response type mismatch / per-connection FIFO violation / pending-queue leak 모두 0이어야 PASS.\n'
|
|
printf -- '- 절대 throughput/latency는 환경 의존 baseline으로 저장하며 고정 합격선으로 쓰지 않는다.\n'
|
|
printf -- '- TypeScript TCP full parallel 1024-client row는 single-process event-loop artifact를 피하기 위해 split-process server topology로 측정하며, row axis에 topology를 기록한다.\n'
|
|
printf -- '- 언어/transport별 미지원 또는 미준비 축은 PASS가 아니라 BLOCKED/SKIPPED와 사유로 기록한다.\n'
|
|
printf -- '- 전체 결과값: `PASS`(측정 범위 내 모든 required 축 stability 0) / `FAIL`(stability 위반) / `INCOMPLETE`(stability 위반은 없으나 기본 선택 범위의 required 축이 SKIP되어 baseline 미완결) / `BLOCKED`(toolchain 미가용).\n'
|
|
printf -- '- `INCOMPLETE`는 PASS가 아니며 lang-baseline 완료 신호로 쓸 수 없다. 명시적으로 범위를 줄여 required 축을 제외하면 그 범위 안에서 PASS가 될 수 있다.\n\n'
|
|
printf '## 언어별 요약\n\n'
|
|
build_lang_summary_table
|
|
printf '\n## 측정 결과\n\n'
|
|
build_rows_table
|
|
printf '\n## SKIPPED/BLOCKED 축\n\n'
|
|
build_skips_table
|
|
printf '\n## git status 요약\n\n'
|
|
printf '```text\n'
|
|
git -C "$repo_root" status --short 2>/dev/null || true
|
|
printf '```\n\n'
|
|
printf '## 언어별 진행 로그(stderr) tail\n\n'
|
|
for lang in $selected_langs; do
|
|
[ -f "$work_dir/${lang}.err" ] || continue
|
|
printf '### %s\n\n' "$lang"
|
|
printf '```text\n'
|
|
tail -n 20 "$work_dir/${lang}.err" 2>/dev/null || true
|
|
printf '```\n\n'
|
|
done
|
|
} >"$record_file"
|
|
|
|
# 사용자 콘솔 요약.
|
|
printf '\n=== stress 측정 결과 (언어별 요약) ===\n' >&2
|
|
build_lang_summary_table >&2
|
|
printf '\n전체 결과값: %s\n' "$overall_result"
|
|
printf '결과 기록 파일: `%s`\n' "${record_file#$repo_root/}"
|
|
|
|
case "$overall_result" in
|
|
PASS) exit 0 ;;
|
|
BLOCKED) exit 3 ;;
|
|
INCOMPLETE) exit 4 ;;
|
|
*) exit 1 ;;
|
|
esac
|