From 1673ba5628074482ea957d835cd5d2e605335c82 Mon Sep 17 00:00:00 2001 From: toki Date: Mon, 8 Jun 2026 19:00:20 +0900 Subject: [PATCH] Fix risk-aware routing classifier output contract for follow-up review --- .../scripts/run_test_by_change.sh | 598 ++++++++++++++++++ 1 file changed, 598 insertions(+) create mode 100755 agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_test_by_change.sh diff --git a/agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_test_by_change.sh b/agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_test_by_change.sh new file mode 100755 index 0000000..75ae3ce --- /dev/null +++ b/agent-ops/skills/project/run-proto-socket-test-matrix/scripts/run_test_by_change.sh @@ -0,0 +1,598 @@ +#!/usr/bin/env bash +set -uo pipefail + +# ----------------------------------------------------------------------- +# run_test_by_change.sh +# 변경 기반 테스트 라우팅 — 분류기 단계 +# +# 마지막 PASS 테스트 record와 현재 작업트리 변경 파일을 기반으로 +# 언어/도메인 분류 결과를 출력한다. 실제 테스트를 실행하지 않는다. +# 후속 라우팅 단계(02+01_risk_routing_rules)가 이 출력을 입력으로 사용한다. +# ----------------------------------------------------------------------- + +usage() { + cat <<'USAGE' +Usage: run_test_by_change.sh [--route auto|speed|stability|full] [--classify-only] [--dry-run] + +마지막 PASS 테스트 record와 현재 변경 파일을 기반으로 분류 결과를 출력한다. +실제 테스트를 실행하지 않는다. + +Options: + --route ROUTE 라우팅 종류: auto, speed, stability, full (기본값: auto) + --classify-only 분류 결과만 출력하고 종료 (현재 지원 모드) + --dry-run 분류/라우팅 결과만 출력하고 종료(명령 실행 없음) + -h, --help 이 도움말 출력 + +Routes: + auto full-matrix, performance quick/full, stress quick/full 중 최신 PASS + speed proto-socket-performance-* PASS/WARN 중 측정 성공 record + stability stress/performance PASS record (stability hard gate 기준) + full proto-socket-full-matrix PASS record +USAGE +} + +route="auto" +classify_only=0 +dry_run=0 + +while [ "$#" -gt 0 ]; do + case "$1" in + --route=*) route="${1#--route=}" ;; + --route) shift; route="${1:-auto}" ;; + --classify-only) classify_only=1 ;; + --dry-run) dry_run=1 ;; + -h|--help) usage; exit 0 ;; + *) + printf 'Error: unknown option "%s"\n' "$1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +case "$route" in + auto|speed|stability|full) ;; + *) + printf 'Error: unknown route "%s". Use: auto, speed, stability, full\n' "$route" >&2 + exit 2 + ;; +esac + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../../../.." && pwd)" +runs_dir="$repo_root/agent-test/runs" + +# ----------------------------------------------------------------------- +# 헬퍼: frontmatter 필드 값 읽기 (--- ... --- 사이의 "field: value" 파싱) +# usage: get_frontmatter_field +# ----------------------------------------------------------------------- +get_frontmatter_field() { + local file="$1" field="$2" + awk -v f="$field" ' + /^---$/ { c++; next } + c == 1 && index($0, f ": ") == 1 { + print substr($0, length(f) + 3) + exit + } + ' "$file" +} + +# ----------------------------------------------------------------------- +# 헬퍼: record 파일에서 git ref 추출 ("- git ref: " 라인) +# ----------------------------------------------------------------------- +get_record_git_ref() { + local file="$1" + awk '/^- git ref: / { print $NF; exit }' "$file" +} + +# ----------------------------------------------------------------------- +# 마지막 PASS record 선택 +# 출력 줄: +# PASS_RECORD:<절대경로 또는 빈 문자열> +# PASS_REF: +# PASS_FALLBACK: +# ----------------------------------------------------------------------- +find_last_pass_record() { + local record="" ref="" fallback="" + + if [ ! -d "$runs_dir" ]; then + printf 'PASS_RECORD:\nPASS_REF:\nPASS_FALLBACK:agent-test/runs/ 디렉터리 없음\n' + return + fi + + # route별 허용 profile 패턴 목록 (공백 구분 문자열) + local patterns + case "$route" in + auto) + patterns="proto-socket-full-matrix proto-socket-performance-quick proto-socket-performance-full proto-socket-stress-quick proto-socket-stress-full" + ;; + speed) + patterns="proto-socket-performance-quick proto-socket-performance-full" + ;; + stability) + patterns="proto-socket-stress-full proto-socket-stress-quick proto-socket-performance-full proto-socket-performance-quick" + ;; + full) + patterns="proto-socket-full-matrix" + ;; + esac + + # 파일명 역순(최신 먼저)으로 탐색 + while IFS= read -r f; do + [ -f "$f" ] || continue + local bname + bname="$(basename "$f")" + + # profile 패턴 매칭 + local matched=0 + for pat in $patterns; do + case "$bname" in + *"$pat"*) matched=1; break ;; + esac + done + [ "$matched" -eq 0 ] && continue + + # overall_result 확인 + local overall + overall="$(get_frontmatter_field "$f" "overall_result")" + + local accept=0 + case "$route" in + speed) + # PASS 또는 WARN (측정 성공) + case "$overall" in PASS|WARN) accept=1 ;; esac + ;; + *) + [ "$overall" = "PASS" ] && accept=1 + ;; + esac + [ "$accept" -eq 0 ] && continue + + # git ref 확인 + local rec_ref + rec_ref="$(get_record_git_ref "$f")" + [ -z "$rec_ref" ] && continue + + # ref 해석 가능 여부 (현재 저장소에서 rev-parse 성공 여부) + if ! git -C "$repo_root" rev-parse --verify "${rec_ref}^{commit}" >/dev/null 2>&1; then + continue + fi + + record="$f" + ref="$rec_ref" + break + done < <(find "$runs_dir" -maxdepth 1 -name '*.md' -type f | sort -r) + + if [ -z "$record" ]; then + fallback="route '${route}'에 맞는 PASS record 없거나 ref 해석 불가 — HEAD 기준 보수 fallback" + fi + + printf 'PASS_RECORD:%s\nPASS_REF:%s\nPASS_FALLBACK:%s\n' \ + "$record" "$ref" "$fallback" +} + +# ----------------------------------------------------------------------- +# 변경 파일 수집 +# tracked diff (last_ref..HEAD) + staged + unstaged + untracked +# archive 경로 제외, 중복 제거, 정렬 +# ----------------------------------------------------------------------- +build_changed_files() { + local last_ref="$1" + { + # 1. tracked: last PASS ref와 HEAD 사이 커밋 변경 + if [ -n "$last_ref" ]; then + git -C "$repo_root" diff --name-only "$last_ref" HEAD 2>/dev/null || true + fi + # 2. staged 변경 + git -C "$repo_root" diff --cached --name-only 2>/dev/null || true + # 3. unstaged 변경 (삭제 파일 포함) + git -C "$repo_root" diff --name-only 2>/dev/null || true + # 4. untracked (gitignored 제외) + git -C "$repo_root" ls-files --others --exclude-standard 2>/dev/null || true + } | sort -u \ + | grep -v '^$' \ + | grep -v '^agent-task/archive/' \ + | grep -v '^agent-roadmap/archive/' \ + || true +} + +# ----------------------------------------------------------------------- +# 파일 하나를 언어와 도메인으로 분류 +# 출력: +# lang: (해당하는 경우) +# domain: (항상 출력) +# +# 도메인 우선순위: proto > perf-harness > test-docs > transport +# > communicator > queue-gateway > agent-ops-internal +# > docs(human-facing tracked) > other +# +# docs-only 판정은 human-facing tracked docs 경로만 대상으로 한다. +# agent-task/**, agent-roadmap/**, agent-ops/**, .tmp/** 는 docs가 아니다. +# ----------------------------------------------------------------------- +classify_file() { + local f="$1" + local lang="" domain="" + + # 언어 분류 + case "$f" in + dart/*) lang="dart" ;; + go/*) lang="go" ;; + kotlin/*) lang="kotlin" ;; + python/*) lang="python" ;; + typescript/*) lang="typescript" ;; + esac + + # proto / schema + case "$f" in + proto/*|*.proto) domain="proto" ;; + esac + + # performance harness + if [ -z "$domain" ]; then + case "$f" in + bench/*|*/bench/*|*/run_performance.sh|*/run_stress.sh) + domain="perf-harness" ;; + esac + fi + + # test docs / runner rules + if [ -z "$domain" ]; then + case "$f" in + agent-test/local/*|agent-ops/skills/project/run-proto-socket-test-matrix/*) + domain="test-docs" ;; + esac + fi + + # transport / crosstest / harness path + if [ -z "$domain" ]; then + case "$f" in + */tcp*|*/ws_*|*/wss*|*/websocket*|*/transport*|*/crosstest*|*/harness*) + domain="transport" ;; + esac + fi + + # communicator / codec / framing / packet / protobuf binding + if [ -z "$domain" ]; then + case "$f" in + */communicator*|*/packet*|*/codec*|*/fram*|*/protobuf*) + domain="communicator" ;; + esac + fi + + # queue / gateway / concurrency + if [ -z "$domain" ]; then + case "$f" in + */queue*|*/gateway*|*/worker*|*/parallel*|*/sustained*|*/stress*) + domain="queue-gateway" ;; + esac + fi + + # agent-ops 운영 파일 — docs-only 판정 대상에서 제외 + # (agent-task, agent-roadmap, agent-ops, .tmp 변경은 테스트 축소 근거가 아님) + if [ -z "$domain" ]; then + case "$f" in + agent-task/*|agent-roadmap/*|agent-ops/*|.tmp/*|.antigravitycli/*) + domain="other" ;; + esac + fi + + # human-facing tracked docs만 docs 분류 + # README.md/PROTOCOL.md/VERSIONING.md/PORTING_GUIDE.md 및 docs/ 하위만 해당 + if [ -z "$domain" ]; then + case "$f" in + README.md|README.*|PORTING_GUIDE.md|PROTOCOL.md|VERSIONING.md|docs/*) + domain="docs" ;; + esac + fi + + # 미분류 + [ -z "$domain" ] && domain="other" + + [ -n "$lang" ] && printf 'lang:%s\n' "$lang" + printf 'domain:%s\n' "$domain" +} + +# ----------------------------------------------------------------------- +# 헬퍼: CSV 문자열에서 항목 존재 확인 +# usage: csv_has_item "" "b" +# ----------------------------------------------------------------------- +csv_has_item() { + local csv="$1" item="$2" + [ -z "$csv" ] && return 1 + case ",${csv}," in + *",${item},"*) return 0 ;; + esac + return 1 +} + +md_escape_cell() { + printf '%s' "$1" | sed 's/|/\\|/g' +} + +# ----------------------------------------------------------------------- +# 라우팅 추천 기록을 위한 상태/중복 제어 헬퍼 +# 추천 상태: +# recommended 실행 후보 +# nightly-pending 별도 시간대 실행 후보 +# skipped-candidate 사용자 라우팅/설정상 미실행 후보 +# ----------------------------------------------------------------------- +add_recommendation() { + local status="$1" + local test_kind="$2" + local target="$3" + local reason="$4" + local key="${status}|${test_kind}|${target}" + + # key 중복 방지 + if [ "${recommend_seen[$key]+x}" = "x" ]; then + return + fi + recommend_seen["$key"]=1 + + reco_statuses+=("$status") + reco_kinds+=("$test_kind") + reco_targets+=("$target") + reco_reasons+=("$reason") +} + +# ----------------------------------------------------------------------- +# route별 후보 정책을 적용해 추천 후보를 생성한다. +# ----------------------------------------------------------------------- +build_recommendations() { + local detected_langs="$1" + local detected_domains="$2" + local file_count="$3" + local docs_only="$4" + local has_perf_term="$5" + + if [ "$file_count" -eq 0 ]; then + add_recommendation "skipped-candidate" "noop" "-" "변경 파일 없음" + return + fi + + # route별 권장 상태: route와 상관없이 후보를 남기되 실행 의도만 바꾼다. + # status=recommended: route가 해당 kind를 기본 의도로 실행 + # status=skipped-candidate: run 조건 미충족으로 추천 제외 + # status=nightly-pending: 별도 시간대/대용량 후보 + local functional_status="skipped-candidate" + local stability_status="skipped-candidate" + local performance_status="skipped-candidate" + local include_performance=0 include_nightly_full=0 + case "$route" in + auto) + functional_status="recommended" + performance_status="recommended" + stability_status="recommended" + include_performance=1 + ;; + speed) + performance_status="recommended" + include_performance=1 + ;; + stability) + stability_status="recommended" + ;; + full) + functional_status="recommended" + performance_status="recommended" + include_performance=1 + include_nightly_full=1 + ;; + esac + + if [ "$docs_only" = "yes" ]; then + add_recommendation "skipped-candidate" "docs-only" "docs" "문서 변경만 감지됨" + if csv_has_item "$detected_domains" "docs" || csv_has_item "$detected_domains" "test-docs"; then + add_recommendation "recommended" "docs-selfcheck" "docs" "문서 변경만 감지된 경우 self-check 후보" + fi + return + fi + + local has_proto has_comm has_transport has_queue has_perf_harness has_test_docs + has_proto=$(csv_has_item "$detected_domains" "proto" && echo 1 || echo 0) + has_comm=$(csv_has_item "$detected_domains" "communicator" && echo 1 || echo 0) + has_transport=$(csv_has_item "$detected_domains" "transport" && echo 1 || echo 0) + has_queue=$(csv_has_item "$detected_domains" "queue-gateway" && echo 1 || echo 0) + has_perf_harness=$(csv_has_item "$detected_domains" "perf-harness" && echo 1 || echo 0) + has_test_docs=$(csv_has_item "$detected_domains" "test-docs" && echo 1 || echo 0) + + # protocol / packet / codec / framing / protobuf 변경 → wider/full functional로 승격 + if [ "$has_proto" -eq 1 ] || [ "$has_comm" -eq 1 ]; then + if [ "$route" = "full" ]; then + add_recommendation "recommended" "functional-full" "all" "proto/communicator 변경은 full functional 범위로 상승" + elif [ "$functional_status" = "recommended" ]; then + add_recommendation "recommended" "functional-wider" "all" "proto/communicator 변경은 smoke로 둬선 안 됨" + else + add_recommendation "skipped-candidate" "functional-wider" "all" "명시 라우팅이 기능 테스트 중심이 아님" + fi + fi + + # 언어 구현 변경: 불명확한 server/client 구분이면 보수적으로 both-x5 권장 + if [ -n "$detected_langs" ]; then + local lang + if [ "$functional_status" = "recommended" ]; then + for lang in $(printf '%s' "$detected_langs" | tr ',' ' '); do + if [ "$route" = "full" ]; then + add_recommendation "recommended" "functional-full" "$lang" "언어별 변경 기반으로 full functional 기준 선택" + else + add_recommendation "recommended" "functional-both-x5" "$lang" "server/client 방향 구분 불명확, 보수적으로 both x5" + fi + done + else + for lang in $(printf '%s' "$detected_langs" | tr ',' ' '); do + add_recommendation "skipped-candidate" "functional-both-x5" "$lang" "명시 라우팅이 full/기능 중심이 아님" + done + fi + fi + + # transport/queue-gateway 변경: functional both + stability quick + if [ "$has_transport" -eq 1 ] || [ "$has_queue" -eq 1 ]; then + add_recommendation "$functional_status" "functional-both-x5" "transport/queue" "transport/queue 변경은 both x5로 확장" + + add_recommendation "$stability_status" "stability-quick" "queue, gateway, concurrency" "transport/queue 변경은 안정성 hard gate 점검 필요" + fi + + # 성능 harness/부하 특성 변경: quick + nightly 후보 분리 + if [ "$has_perf_harness" -eq 1 ] || [ "$has_perf_term" -eq 1 ]; then + if [ "$performance_status" = "recommended" ]; then + add_recommendation "recommended" "performance-quick" "performance baseline" "성능 harness 또는 부하 시나리오 변경으로 빠른 성능 신호가 필요" + else + add_recommendation "skipped-candidate" "performance-quick" "performance baseline" "명시 라우팅이 성능 중심이 아님" + fi + if [ "$include_performance" -eq 1 ] && [ "$performance_status" = "recommended" ]; then + add_recommendation "nightly-pending" "performance-full" "performance baseline" "sustained/high parallel/payload heavy 지표는 야간 full 후보" + fi + fi + + # 안정성 full/long-run 후보는 요청/경로와 무관하게 후보만 남김 + if [ "$has_queue" -eq 1 ] || [ "$has_transport" -eq 1 ]; then + if [ "$include_nightly_full" -eq 1 ]; then + add_recommendation "nightly-pending" "stability-full" "queue/gateway/transport" "장시간 백로그/누수 확인은 야간 후보로 분리" + fi + fi + + # test docs 변경은 해당 라우팅 계약 self-check 대상 + if [ "$has_test_docs" -eq 1 ]; then + local test_docs_status="skipped-candidate" + [ "$route" = "auto" ] && test_docs_status="recommended" + add_recommendation "$test_docs_status" "runner-docs-selfcheck" "test-docs" "runner/문서 변경은 self-check 또는 dry-run 후보" + fi + + if [ "${#reco_statuses[@]}" -eq 0 ]; then + add_recommendation "skipped-candidate" "unknown" "-" "변경 분류 기준상 명시 라우팅 대상이 없음" + fi +} + +# ----------------------------------------------------------------------- +# 메인 +# ----------------------------------------------------------------------- +main() { + # 1. 마지막 PASS record 탐색 + local pass_info + pass_info="$(find_last_pass_record)" + local last_record last_ref fallback_reason + last_record="$(printf '%s\n' "$pass_info" | grep '^PASS_RECORD:' | sed 's/^PASS_RECORD://')" + last_ref="$(printf '%s\n' "$pass_info" | grep '^PASS_REF:' | sed 's/^PASS_REF://')" + fallback_reason="$(printf '%s\n' "$pass_info" | grep '^PASS_FALLBACK:' | sed 's/^PASS_FALLBACK://')" + + # 2. 변경 파일 수집 + local changed_files + changed_files="$(build_changed_files "$last_ref")" + + local file_count=0 + if [ -n "$changed_files" ]; then + file_count="$(printf '%s\n' "$changed_files" | grep -c . || true)" + fi + + # 3. 파일별 언어/도메인 분류 집계 + local tmp_lang tmp_domain + tmp_lang="$(mktemp)" + tmp_domain="$(mktemp)" + local file_table="" has_non_docs=0 has_performance_term=0 + + if [ -n "$changed_files" ]; then + while IFS= read -r f; do + [ -z "$f" ] && continue + local cls lang_val domain_val + cls="$(classify_file "$f")" + lang_val="$(printf '%s\n' "$cls" | grep '^lang:' | sed 's/^lang://' || true)" + domain_val="$(printf '%s\n' "$cls" | grep '^domain:' | sed 's/^domain://' || true)" + + [ -n "$lang_val" ] && printf '%s\n' "$lang_val" >> "$tmp_lang" + [ -n "$domain_val" ] && printf '%s\n' "$domain_val" >> "$tmp_domain" + case "${domain_val:-}" in + docs|test-docs) ;; + *) has_non_docs=1 ;; + esac + + file_table="${file_table}| $(md_escape_cell "$f") | $(md_escape_cell "${lang_val:-}") | $(md_escape_cell "${domain_val:-}") |"$'\n' + case "$f" in + *payload*|*sustained*|*parallel*|*bench*|*run_performance*|*run_stress*|*run_performance.sh|*run_stress.sh) + case "$domain_val" in + perf-harness|communicator|transport|queue-gateway|proto) + has_performance_term=1 + ;; + esac + ;; + esac + done <<< "$changed_files" + fi + + # 중복 제거 + 쉼표 결합 + local detected_langs="" detected_domains="" + if [ -s "$tmp_lang" ]; then + detected_langs="$(sort -u "$tmp_lang" | tr '\n' ',' | sed 's/,$//')" + fi + if [ -s "$tmp_domain" ]; then + detected_domains="$(sort -u "$tmp_domain" | tr '\n' ',' | sed 's/,$//')" + fi + rm -f "$tmp_lang" "$tmp_domain" + + # docs-only 판정: 변경 파일이 있고 docs 외 domain이 없으면 docs-only + local docs_only="no" + [ "$has_non_docs" -eq 0 ] && [ "$file_count" -gt 0 ] && docs_only="yes" + + local -A recommend_seen=() + declare -a reco_statuses=() + declare -a reco_kinds=() + declare -a reco_targets=() + declare -a reco_reasons=() + if [ "$classify_only" -eq 0 ]; then + build_recommendations "$detected_langs" "$detected_domains" "$file_count" "$docs_only" "$has_performance_term" + fi + + # untracked 포함 여부 + local untracked_included="no" + if git -C "$repo_root" ls-files --others --exclude-standard 2>/dev/null | grep -q .; then + untracked_included="yes" + fi + + # ---- 결과 출력 ---- + printf '## 변경 기반 분류 결과\n\n' + printf 'route: %s\n' "$route" + + if [ -n "$last_record" ]; then + local rel_record + rel_record="$(realpath --relative-to="$repo_root" "$last_record" 2>/dev/null \ + || printf '%s' "$(basename "$last_record")")" + printf 'last_pass_record: %s\n' "$rel_record" + printf 'last_pass_ref: %s\n' "$last_ref" + printf 'fallback_reason: (없음)\n' + else + printf 'last_pass_record: (없음)\n' + printf 'last_pass_ref: (없음)\n' + printf 'fallback_reason: %s\n' "${fallback_reason:-(알 수 없음)}" + fi + + printf 'changed_file_count: %s\n' "$file_count" + printf 'detected_languages: %s\n' "${detected_langs:-없음}" + printf 'detected_domains: %s\n' "${detected_domains:-없음}" + printf 'docs_only: %s\n' "$docs_only" + printf 'untracked_included: %s\n' "$untracked_included" + + printf '\n### 변경 파일 목록\n\n' + if [ "$file_count" -eq 0 ]; then + printf '변경 파일 없음\n' + else + printf '| 파일 | 언어 | 도메인 |\n' + printf '|------|------|--------|\n' + printf '%s' "$file_table" + fi + + if [ "$classify_only" -eq 0 ]; then + printf '\n### 라우팅 추천 결과\n\n' + printf '| 상태 | 테스트 후보 | 대상 | 근거 |\n' + printf '|---|---|---|---|\n' + local i + for ((i = 0; i < ${#reco_statuses[@]}; i++)); do + printf '| %s | %s | %s | %s |\n' \ + "$(md_escape_cell "${reco_statuses[$i]}")" \ + "$(md_escape_cell "${reco_kinds[$i]}")" \ + "$(md_escape_cell "${reco_targets[$i]}")" \ + "$(md_escape_cell "${reco_reasons[$i]}")" + done + fi +} + +main