From e74b50c7f387206c2dad91219eec74e571d236ec Mon Sep 17 00:00:00 2001 From: toki Date: Mon, 1 Jun 2026 17:41:55 +0900 Subject: [PATCH] feat: persistent executor output filtering and test updates - Add output filtering for persistent CLI executor - Update e2e smoke test script and skill documentation - Update testing domain rules --- .../rules/project/domain/testing/rules.md | 4 +- agent-ops/skills/project/e2e-smoke/SKILL.md | 8 +- apps/node/internal/adapters/cli/persistent.go | 67 +++- .../cli/persistent_execute_blackbox_test.go | 161 +++++++++ .../adapters/cli/persistent_output_filter.go | 89 ++++- .../cli/persistent_output_filter_test.go | 71 ++++ scripts/e2e-smoke.sh | 309 ++++++++++++++---- scripts/fixtures/user-e2e-prompts.tsv | 11 + 8 files changed, 629 insertions(+), 91 deletions(-) create mode 100644 scripts/fixtures/user-e2e-prompts.tsv diff --git a/agent-ops/rules/project/domain/testing/rules.md b/agent-ops/rules/project/domain/testing/rules.md index a4336a6..964957e 100644 --- a/agent-ops/rules/project/domain/testing/rules.md +++ b/agent-ops/rules/project/domain/testing/rules.md @@ -51,8 +51,8 @@ last_rule_updated_at: 2026-06-01 - 사용자 실행 파이프라인에 닿는 작업을 한 경우, 작업 완료 후 일반 Go 테스트와 변경 범위에 맞는 full-cycle 실제 구동을 검증한다. repo 내부 edge-node 진단에는 `scripts/dev/edge.sh` + `scripts/dev/node.sh`를 사용할 수 있지만, field 사용자 기본 UX는 bootstrap 명령으로 확인한다. - 사용자 실행 파이프라인에는 `scripts/dev/**`, `scripts/e2e-*.sh`, `Makefile`, `docker-compose.yml`, `apps/*/cmd/**`, `apps/*/internal/bootstrap/**`, edge-node transport/service/registry/input surface, Control Plane wire endpoint, client wire surface, adapter 실행/stream/cancel/status 경로, `configs/**`, `packages/go/config/**`, `packages/go/hostsetup/**`, 관련 protobuf 계약 변경이 포함된다. - edge-node 개발 진단 흐름 검증은 `scripts/dev/edge.sh`와 `scripts/dev/node.sh`를 별도 프로세스로 직접 실행하고, edge console prompt에 명령을 한 줄씩 입력한 뒤 기대 출력이 도착한 것을 확인하고 다음 입력으로 넘어간다. -- 메시지 검증 기준은 edge console에서 같은 session으로 메시지 2회를 보내고, 각 요청마다 `[edge] sent`, `[node-*-event] start`, 비어 있지 않은 `[node-*-message]`, `[node-*-event] complete`가 edge 화면에 표시되는 것이다. -- 완료 이벤트만으로 정상 판정하지 않는다. node 로컬 출력에 생성된 `[node-message]` payload가 edge console의 `[node-*-message]` 출력에 모두 표시되어야 하며, complete event는 모든 message payload가 edge에 도착한 뒤의 마감 신호로 본다. +- 메시지 검증 기준은 edge console에서 같은 session으로 메시지 2회를 보내고, 각 요청마다 `[edge] sent`, `[node-*-event] start`, 기대 payload를 포함한 비어 있지 않은 `[node-*-message]`, 같은 run의 `[node-*-event] complete`가 edge 화면에 표시되는 것이다. +- 완료 이벤트만으로 정상 판정하지 않는다. `[edge] sent`나 `[edge-message]`에 보이는 사용자 입력 echo는 응답 relay 증거가 아니며, node 로컬 출력에 생성된 같은 run의 `[node-message]` payload 라인 목록이 edge console의 `[node-*-message]` 라인 목록과 내용/순서까지 동일해야 한다. complete event는 동일성 확인 대상인 모든 message payload가 edge에 표시된 뒤의 마감 신호이며, 답변이 끝나기 전에 complete가 먼저 표시되거나 complete 뒤에 같은 run의 message가 추가로 표시되면 실패다. - command 검증 기준은 edge console에서 `/nodes`와 변경 범위에 닿는 command를 직접 입력하고, node에서 온 결과가 edge 화면에 `[node-*-]` 또는 명확한 성공/unsupported/error 출력으로 표시되는 것이다. CLI 경로 변경 시 최소 `/capabilities`, `/transport`, `/sessions`, persistent profile이면 `/terminate-session`을 확인한다. - 보조 E2E smoke는 mock adapter와 임시 설정/포트를 사용해 외부 CLI 의존성 없이 수행한다. - 보조 E2E smoke에서는 최소한 node 등록, `/nodes` 확인, console 메시지 전송, delta/message 출력, complete event를 확인한다. diff --git a/agent-ops/skills/project/e2e-smoke/SKILL.md b/agent-ops/skills/project/e2e-smoke/SKILL.md index 62c7f52..3c17a84 100644 --- a/agent-ops/skills/project/e2e-smoke/SKILL.md +++ b/agent-ops/skills/project/e2e-smoke/SKILL.md @@ -54,8 +54,9 @@ repo 내부 edge-node 진단이 필요할 때 `scripts/dev/edge.sh`와 `scripts/ 4. **메시지 2회 왕복 기준** - edge console prompt에 메시지를 한 번 입력한다. - 첫 번째 요청에서 `[edge] sent`, `[node-*-event] start`, 비어 있지 않은 `[node-*-message]`, `[node-*-event] complete`가 edge 화면에 도착해야 한다. - - node 로컬 출력에 생성된 `[node-message]` payload가 edge console의 `[node-*-message]` 출력에 모두 표시되었는지 비교한다. - - complete event만으로 정상 판정하지 않는다. complete event는 node가 생성한 모든 message payload가 edge에 표시된 뒤의 마감 신호여야 한다. + - node 로컬 출력에 생성된 같은 run의 `[node-message]` payload 라인 목록이 edge console의 `[node-*-message]` payload 라인 목록과 내용/순서까지 동일한지 비교한다. + - complete event만으로 정상 판정하지 않는다. complete event는 node가 생성한 모든 message payload가 edge에 동일하게 표시된 뒤의 마감 신호여야 한다. + - 답변이 끝나기 전에 edge complete가 먼저 표시되거나 complete 뒤에 같은 run의 message가 추가로 표시되면 실패로 판정한다. - 같은 session에서 두 번째 메시지를 입력하고 같은 출력 흐름이 다시 도착해야 한다. - 두 번째 메시지가 첫 번째 메시지 이후에도 같은 edge-node 연결에서 정상 처리되는지 확인한다. - foreground run에서는 complete event 전에 다음 검증으로 넘어가지 않는다. @@ -135,7 +136,8 @@ terminated session default node=test-node - [ ] `scripts/dev/edge.sh`와 `scripts/dev/node.sh`를 각각 실행한 repo 내부 edge-node 진단 결과가 보고되었는가 - [ ] node register와 `/nodes` 결과가 edge 화면에 표시되었는가 - [ ] 같은 session에서 메시지 2회가 각각 sent/start/message/complete로 표시되었는가 -- [ ] node 로컬 `[node-message]` payload가 edge `[node-*-message]` 출력에 모두 표시되었는가 +- [ ] node 로컬 `[node-message]` payload 라인 목록이 edge `[node-*-message]` 출력과 run별로 내용/순서까지 동일한가 +- [ ] edge complete가 같은 run의 마지막 message 출력 이후에 표시되고, complete 뒤에 같은 run의 message가 추가로 표시되지 않았는가 - [ ] `/capabilities`, `/transport`, `/sessions`, 관련 command 결과가 edge 화면에 표시되었는가 - [ ] 보조 smoke를 실행했다면 repo 내부 edge-node 진단 또는 field bootstrap 검증과 구분해 보고했는가 - [ ] 풀테스트에서 실제 외부 CLI profile별 메시지 왕복과 `/status` 결과가 보고되었는가 diff --git a/apps/node/internal/adapters/cli/persistent.go b/apps/node/internal/adapters/cli/persistent.go index 94d5a4e..db48b40 100644 --- a/apps/node/internal/adapters/cli/persistent.go +++ b/apps/node/internal/adapters/cli/persistent.go @@ -14,6 +14,7 @@ import ( "github.com/creack/pty" "go.uber.org/zap" + "iop/apps/node/internal/adapters/cli/status" "iop/apps/node/internal/runtime" "iop/packages/go/config" ) @@ -104,6 +105,7 @@ func (c *CLI) executePersistent(ctx context.Context, spec runtime.ExecutionSpec, }() outputTokens := 0 var markerBuf strings.Builder + claudePromptReplayCount := 0 for { select { @@ -122,6 +124,19 @@ func (c *CLI) executePersistent(ctx context.Context, spec runtime.ExecutionSpec, err := drainPersistentDone(sess) return c.emitPersistentExit(ctx, sink, spec.RunID, targetName, profile, sess, err) } + if waitForFilteredMessage { + if msg, cancelled := claudeTerminalCancelMessage(sess.getTail()); cancelled { + c.removePersistentSession(sess) + _ = closeProfileSession(context.Background(), sess) + _ = sink.Emit(ctx, runtime.RuntimeEvent{ + RunID: spec.RunID, + Type: runtime.EventTypeCancelled, + Message: msg, + Timestamp: time.Now(), + }) + return fmt.Errorf("%w: %s", runtime.ErrRunCancelled, msg) + } + } if delta := outputFilter.Filter(out.text); delta != "" { outputTokens += len(strings.Fields(delta)) _ = sink.Emit(ctx, runtime.RuntimeEvent{ @@ -180,6 +195,13 @@ func (c *CLI) executePersistent(ctx context.Context, spec runtime.ExecutionSpec, Timestamp: time.Now(), }) } else if waitForFilteredMessage && !outputFilter.HasOutput() { + if claudePromptReplayCount == 0 && claudeTerminalReadyForInput(sess.getTail()) { + claudePromptReplayCount++ + c.logger.Info("cli adapter: replaying claude prompt after ready screen", zap.String("target", targetName), zap.String("session", sess.key.sessionID)) + if err := writePrompt(ctx, sess.input, prompt, profile); err != nil { + return emitRuntimeError(ctx, sink, spec.RunID, fmt.Sprintf("replay prompt: %v", err)) + } + } idleTimer.Reset(idleTimeout) continue } @@ -474,7 +496,38 @@ func shouldAcceptClaudeBypassWarning(raw string) bool { strings.Contains(compact, "yesiaccept") } +func claudeTerminalCancelMessage(raw string) (string, bool) { + compact := compactTerminalText(raw) + switch { + case strings.Contains(compact, "youvehityoursessionlimit"): + if line := findClaudeTerminalLine(raw, "session limit"); line != "" { + return line, true + } + return "Claude session limit reached", true + case strings.Contains(compact, "stopandwaitforlimittoreset") && strings.Contains(compact, "upgradeyourplan"): + return "Claude session limit reached; upgrade was not selected", true + default: + return "", false + } +} + +func claudeTerminalReadyForInput(raw string) bool { + return claudeScreenHasInputPrompt(status.RenderVisibleScreen(raw, terminalRows, terminalCols)) +} + +func findClaudeTerminalLine(raw, needle string) string { + needle = strings.ToLower(needle) + for _, line := range strings.Split(cleanClaudeTerminalOutput(raw), "\n") { + line = strings.Join(strings.Fields(line), " ") + if strings.Contains(strings.ToLower(line), needle) { + return line + } + } + return "" +} + func compactTerminalText(s string) string { + s = cleanClaudeTerminalOutput(s) var b strings.Builder inEscape := false for _, r := range s { @@ -528,11 +581,7 @@ func drainPersistentDone(sess *profileSession) error { } func (c *CLI) emitPersistentExit(ctx context.Context, sink runtime.EventSink, runID, targetName string, profile config.CLIProfileConf, sess *profileSession, err error) error { - c.mu.Lock() - if s, found := c.sessions[sess.key]; found && s == sess { - delete(c.sessions, sess.key) - } - c.mu.Unlock() + c.removePersistentSession(sess) cmdSummary := fmt.Sprintf("%s %s", profile.Command, strings.Join(profile.Args, " ")) tail := sess.getTail() @@ -549,3 +598,11 @@ func (c *CLI) emitPersistentExit(ctx context.Context, sink runtime.EventSink, ru } return emitRuntimeError(ctx, sink, runID, msg) } + +func (c *CLI) removePersistentSession(sess *profileSession) { + c.mu.Lock() + if s, found := c.sessions[sess.key]; found && s == sess { + delete(c.sessions, sess.key) + } + c.mu.Unlock() +} diff --git a/apps/node/internal/adapters/cli/persistent_execute_blackbox_test.go b/apps/node/internal/adapters/cli/persistent_execute_blackbox_test.go index a9cbbf2..cc58c4e 100644 --- a/apps/node/internal/adapters/cli/persistent_execute_blackbox_test.go +++ b/apps/node/internal/adapters/cli/persistent_execute_blackbox_test.go @@ -2,6 +2,7 @@ package cli_test import ( "context" + "errors" "fmt" "os" osexec "os/exec" @@ -326,6 +327,105 @@ func TestCLIExecutePersistentTerminalAcceptsClaudeWorkspaceTrustAndBypassWarning } } +func TestCLIExecutePersistentClaudeTUICancelsSessionLimitUpgradePrompt(t *testing.T) { + testutil.RequirePTYSupport(t) + if _, err := osexec.LookPath("stty"); err != nil { + t.Skip("stty required") + } + + cfg := config.CLIConf{ + Enabled: true, + Profiles: map[string]config.CLIProfileConf{ + "claude-tui": { + Command: os.Args[0], + Args: []string{"-test.run=TestRawTUIHelperProcess", "--"}, + Env: []string{"IOP_CLAUDE_RATE_LIMIT_HELPER=1"}, + Persistent: true, + Terminal: true, + ResponseIdleTimeoutMS: 500, + StartupIdleTimeoutMS: 50, + }, + }, + } + c := clipkg.New(cfg, zap.NewNop()) + + ctx := context.Background() + if err := c.Start(ctx); err != nil { + t.Fatalf("start: %v", err) + } + defer func() { _ = c.Stop(ctx) }() + + execCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + sink := &testutil.FakeSink{} + err := c.Execute(execCtx, noderuntime.ExecutionSpec{ + RunID: "run-claude-rate-limit", + Target: "claude-tui", + Input: map[string]any{"prompt": "hello"}, + }, sink) + if !errors.Is(err, noderuntime.ErrRunCancelled) { + t.Fatalf("expected run cancelled, got %v", err) + } + var cancelMsg string + for _, ev := range sink.Events() { + if ev.Type == noderuntime.EventTypeCancelled { + cancelMsg = ev.Message + } + if ev.Type == noderuntime.EventTypeError { + t.Fatalf("expected cancellation, got error event %q", ev.Error) + } + } + if !strings.Contains(cancelMsg, "session limit") { + t.Fatalf("expected cancelled event to mention session limit, got %q", cancelMsg) + } +} + +func TestCLIExecutePersistentClaudeTUIReplaysPromptAfterStartupReadyRace(t *testing.T) { + testutil.RequirePTYSupport(t) + if _, err := osexec.LookPath("stty"); err != nil { + t.Skip("stty required") + } + + cfg := config.CLIConf{ + Enabled: true, + Profiles: map[string]config.CLIProfileConf{ + "claude-tui": { + Command: os.Args[0], + Args: []string{"-test.run=TestRawTUIHelperProcess", "--"}, + Env: []string{"IOP_CLAUDE_READY_REPLAY_HELPER=1"}, + Persistent: true, + Terminal: true, + ResponseIdleTimeoutMS: 100, + StartupIdleTimeoutMS: 40, + }, + }, + } + c := clipkg.New(cfg, zap.NewNop()) + + ctx := context.Background() + if err := c.Start(ctx); err != nil { + t.Fatalf("start: %v", err) + } + defer func() { _ = c.Stop(ctx) }() + + execCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + sink := &testutil.FakeSink{} + err := c.Execute(execCtx, noderuntime.ExecutionSpec{ + RunID: "run-claude-ready-race", + Target: "claude-tui", + Input: map[string]any{"prompt": "hello"}, + }, sink) + if err != nil { + t.Fatalf("execute: %v", err) + } + if combined := testutil.CollectDeltas(sink.Events()); combined != "replay:hello" { + t.Fatalf("expected replayed prompt response, got %q", combined) + } +} + func TestRawTUIHelperProcess(t *testing.T) { switch { case os.Getenv("IOP_RAW_TUI_HELPER") == "1": @@ -334,6 +434,10 @@ func TestRawTUIHelperProcess(t *testing.T) { runClaudeWarningHelper() case os.Getenv("IOP_CLAUDE_TRUST_WARNING_HELPER") == "1": runClaudeTrustAndWarningHelper() + case os.Getenv("IOP_CLAUDE_RATE_LIMIT_HELPER") == "1": + runClaudeRateLimitHelper() + case os.Getenv("IOP_CLAUDE_READY_REPLAY_HELPER") == "1": + runClaudeReadyReplayHelper() default: return } @@ -426,6 +530,63 @@ func runClaudeTrustAndWarningHelper() { readPromptAndReply() } +func runClaudeRateLimitHelper() { + setRawTerminal() + for { + buf := make([]byte, 1) + n, err := os.Stdin.Read(buf) + if n > 0 && buf[0] == '\r' { + fmt.Fprint(os.Stdout, "\u25cf \u23bf You've hit your session limit \u00b7 resets 12:50pm (Asia/Seoul)\r\n") + fmt.Fprint(os.Stdout, "What do you want to do?\r\n❯ 1. Stop and wait for limit to reset\r\n2. Upgrade your plan\r\n") + _ = os.Stdout.Sync() + } + if err != nil { + os.Exit(0) + } + } +} + +func runClaudeReadyReplayHelper() { + setRawTerminal() + fmt.Fprint(os.Stdout, "Quick safety check: Is this a project you created or one you trust?\r\n❯ 1. Yes, I trust this folder\r\n2. No, exit\r\n") + _ = os.Stdout.Sync() + + _ = readRawLine() + time.Sleep(150 * time.Millisecond) + _ = readRawLine() + + fmt.Fprint(os.Stdout, "\r\n❯ ") + _ = os.Stdout.Sync() + + prompt := readRawLine() + fmt.Fprintf(os.Stdout, "\r\n\u25cf replay:%s\r\n\u276f ", prompt) + _ = os.Stdout.Sync() + + buf := make([]byte, 1) + for { + if _, err := os.Stdin.Read(buf); err != nil { + os.Exit(0) + } + } +} + +func readRawLine() string { + var line []byte + buf := make([]byte, 1) + for { + n, err := os.Stdin.Read(buf) + if n > 0 { + if buf[0] == '\r' || buf[0] == '\n' { + return string(line) + } + line = append(line, buf[0]) + } + if err != nil { + os.Exit(0) + } + } +} + func readPromptAndReply() { var prompt []byte buf := make([]byte, 1) diff --git a/apps/node/internal/adapters/cli/persistent_output_filter.go b/apps/node/internal/adapters/cli/persistent_output_filter.go index fa7fde7..d8ff01d 100644 --- a/apps/node/internal/adapters/cli/persistent_output_filter.go +++ b/apps/node/internal/adapters/cli/persistent_output_filter.go @@ -94,14 +94,23 @@ func (f *claudeTUIOutputFilter) currentMessage() claudeMessageSnapshot { screen := status.RenderVisibleScreen(f.raw.String(), terminalRows, terminalCols) clean := cleanClaudeTerminalOutput(f.raw.String()) message, closed := latestClaudeAssistantMessage(screen) - if cleanMessage, cleanClosed := latestClaudeAssistantMessageFromCleanOutput(clean); cleanMessage != "" { - if message == "" { - f.seenAssistant = true - return claudeMessageSnapshot{text: cleanMessage, closed: cleanClosed} + cleanMessage, cleanClosed := latestClaudeAssistantMessageFromCleanOutput(clean) + if cleanMessage != "" { + if isClaudePromptEchoMessage(cleanMessage, f.prompt) { + cleanMessage = "" } + } + if cleanMessage != "" { + f.seenAssistant = true if cleanMessage == message { closed = closed || cleanClosed + return claudeMessageSnapshot{text: message, closed: closed} } + return claudeMessageSnapshot{text: cleanMessage, closed: cleanClosed} + } + if isClaudePromptEchoMessage(message, f.prompt) { + message = "" + closed = false } if message != "" { f.seenAssistant = true @@ -248,12 +257,18 @@ func appendClaudeMessageLine(out *[]string, line string) { } func claudeAssistantLine(line string) (string, bool) { - const marker = "\u25cf" - idx := strings.Index(line, marker) - if idx < 0 { - return "", false + for _, marker := range []string{"\u25cf", "\u23fa"} { + idx := strings.Index(line, marker) + if idx < 0 { + continue + } + text := strings.TrimSpace(line[idx+len(marker):]) + if text == "" || isClaudeTUIChromeLine(text) { + return "", false + } + return text, true } - return strings.TrimSpace(line[idx+len(marker):]), true + return "", false } func isClaudeTUIBoundaryLine(line string) bool { @@ -289,6 +304,8 @@ func isClaudeTUIBoundaryLine(line string) bool { return true case strings.Contains(lower, "shift+tab to cycle"): return true + case strings.Contains(lower, "/effort"): + return true case strings.Contains(lower, "bypass permissions on"): return true case strings.Contains(lower, "for agents"): @@ -301,6 +318,60 @@ func isClaudeTUIBoundaryLine(line string) bool { return false } +func isClaudeTUIChromeLine(line string) bool { + line = strings.TrimSpace(line) + if line == "" { + return true + } + lower := strings.ToLower(line) + switch { + case strings.Contains(lower, "/effort"): + return true + case strings.Contains(lower, "esc to interrupt"): + return true + case isClaudeTransientStatusLine(line): + return true + } + return false +} + +func isClaudePromptEchoMessage(message, prompt string) bool { + prompt = normalizeClaudePromptEchoText(prompt) + if prompt == "" { + return false + } + lines := strings.Split(message, "\n") + checked := 0 + for _, line := range lines { + rawLine := strings.TrimSpace(line) + if strings.Contains(rawLine, "_") && len(rawLine) >= 12 { + return false + } + normalizedLine := normalizeClaudePromptEchoText(rawLine) + if normalizedLine == "" { + continue + } + if len(normalizedLine) >= 16 { + return false + } + if !strings.Contains(prompt, normalizedLine) { + return false + } + checked++ + } + return checked > 0 +} + +func normalizeClaudePromptEchoText(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + func fallbackClaudeVisibleMessage(text, prompt string) string { lines := strings.Split(text, "\n") var block []string diff --git a/apps/node/internal/adapters/cli/persistent_output_filter_test.go b/apps/node/internal/adapters/cli/persistent_output_filter_test.go index dfc4c1a..30596e8 100644 --- a/apps/node/internal/adapters/cli/persistent_output_filter_test.go +++ b/apps/node/internal/adapters/cli/persistent_output_filter_test.go @@ -183,6 +183,77 @@ func TestClaudeTUIOutputFilterKeepsAssistantAfterTransientStatus(t *testing.T) { } } +func TestClaudeTUIOutputFilterIgnoresEffortStatusChrome(t *testing.T) { + filter := newClaudeTUIOutputFilter("hello") + + filter.Filter("\x1b[2C\u25cf high\u00b7/effort\r\n") + filter.Filter("in\u2026\r\n\r\n\r\n\r\n\r\ng\r\n\u276f ") + + if got := filter.Flush(); got != "" { + t.Fatalf("Flush() = %q, want no chrome output", got) + } + if filter.HasOutput() { + t.Fatalf("HasOutput() = true, want false for chrome-only screen") + } +} + +func TestClaudeTUIOutputFilterAcceptsRecordCircleAssistantMarker(t *testing.T) { + filter := newClaudeTUIOutputFilter("hello") + + got := filter.Filter("\u23faIOP_TUI_MARKER_OK\r\n\u2733 Doodling\u2026 (1s \u00b7 \u2193 1 tokens)\r\n\u276f ") + if got != "IOP_TUI_MARKER_OK" { + t.Fatalf("Filter() = %q, want %q", got, "IOP_TUI_MARKER_OK") + } +} + +func TestClaudeTUIOutputFilterSuppressesPromptEchoFragments(t *testing.T) { + filter := newClaudeTUIOutputFilter("Reply with exactly and only this token: IOP_TUI_MARKER_OK") + + for _, chunk := range []string{ + "\u25cf R\r\n", + "\u25cf eplywithex\r\n", + "\u25cf actly\r\n", + "\u25cf token:\r\n", + } { + if got := filter.Filter(chunk); got != "" { + t.Fatalf("Filter(%q) = %q, want prompt echo suppressed", chunk, got) + } + } + + got := filter.Filter("\u23faIOP_TUI_MARKER_OK\r\n\u276f ") + if got != "IOP_TUI_MARKER_OK" { + t.Fatalf("Filter() = %q, want assistant token", got) + } +} + +func TestClaudeStartupPromptDetectionIgnoresCursorPositioning(t *testing.T) { + rawTrust := "\x1b[2GQuick\x1b[8Gsafety\x1b[15Gcheck:\x1b[22GIs\x1b[25Gthis\x1b[30Ga\x1b[32Gproject\r\n" + + "\x1b[2G\x1b[38;5;153m❯\x1b[4G\x1b[38;5;246m1.\x1b[7G\x1b[38;5;153mYes,\x1b[12GI\x1b[14Gtrust\x1b[20Gthis\x1b[25Gfolder\x1b[39m\r\n" + if !shouldAcceptClaudeWorkspaceTrust(rawTrust) { + t.Fatalf("workspace trust prompt with cursor positioning was not detected") + } + + rawBypass := "\x1b[2GWARNING:\x1b[11GClaude\x1b[18GCode\x1b[23Grunning\x1b[31Gin\x1b[34GBypassPermissions\x1b[52Gmode\r\n" + + "\x1b[4G2.\x1b[7GYes,\x1b[12GI\x1b[14Gaccept\r\n" + if !shouldAcceptClaudeBypassWarning(rawBypass) { + t.Fatalf("bypass warning with cursor positioning was not detected") + } +} + +func TestClaudeTerminalCancelMessageDetectsSessionLimit(t *testing.T) { + raw := "\x1b[2G⎿ \u00a0You've hit your session limit · resets 12:50pm (Asia/Seoul)\r\n" + + "\x1b[2GWhat do you want to do?\r\n" + + "\x1b[2G❯\x1b[4G1.\x1b[7GStop\x1b[12Gand\x1b[16Gwait\x1b[21Gfor\x1b[25Glimit\x1b[31Gto\x1b[34Greset\r\n" + + "\x1b[4G2.\x1b[7GUpgrade\x1b[15Gyour\x1b[20Gplan\r\n" + msg, ok := claudeTerminalCancelMessage(raw) + if !ok { + t.Fatalf("session limit cancellation was not detected") + } + if !strings.Contains(msg, "session limit") { + t.Fatalf("cancel message = %q, want session limit detail", msg) + } +} + func testProfile(command string, terminal bool) config.CLIProfileConf { return config.CLIProfileConf{Command: command, Terminal: terminal} } diff --git a/scripts/e2e-smoke.sh b/scripts/e2e-smoke.sh index b8258d7..c6bc117 100755 --- a/scripts/e2e-smoke.sh +++ b/scripts/e2e-smoke.sh @@ -14,7 +14,37 @@ else fi TMP_DIR=$(mktemp -d) -trap 'rm -rf "$TMP_DIR" 2>/dev/null || true' EXIT + +kill_process_tree() { + local pid="$1" + local child + + if [ -z "$pid" ]; then + return 0 + fi + + if command -v pgrep >/dev/null 2>&1; then + while IFS= read -r child; do + if [ -n "$child" ]; then + kill_process_tree "$child" + fi + done </dev/null || true) +EOF_CHILDREN + fi + + kill "$pid" 2>/dev/null || true +} + +cleanup() { + set +e + exec 3>&- 2>/dev/null || true + kill_process_tree "${NODE_PID:-}" + kill_process_tree "${EDGE_PID:-}" + rm -rf "$TMP_DIR" 2>/dev/null || true +} + +trap cleanup EXIT EDGE_CONFIG="$TMP_DIR/edge.yaml" NODE_CONFIG="$TMP_DIR/node.yaml" @@ -33,15 +63,51 @@ COMMAND_SETTLE_SECONDS="${IOP_E2E_COMMAND_SETTLE_SECONDS:-0.2}" IS_PERSISTENT=0 HAS_STATUS=0 EXPECT_TAIL_MESSAGES=0 -FIRST_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_alpha" -FIRST_EXPECTED="SMOKE_ALPHA" -FIRST_TAIL_EXPECTED="IOP_SMOKE_ALPHA_TAIL" -SECOND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_beta" -SECOND_EXPECTED="SMOKE_BETA" -SECOND_TAIL_EXPECTED="IOP_SMOKE_BETA_TAIL" -BACKGROUND_PROMPT="Convert this token to uppercase and reply with only the converted token: iop_smoke_bg" -BACKGROUND_EXPECTED="SMOKE_BG" -BACKGROUND_TAIL_EXPECTED="IOP_SMOKE_BG_TAIL" +PROMPT_TEMPLATE_FILE="${IOP_E2E_PROMPT_TEMPLATES:-$REPO_ROOT/scripts/fixtures/user-e2e-prompts.tsv}" + +prompt_template_count() { + awk -F'|' 'NF >= 3 && $1 !~ /^#/ { n++ } END { print n + 0 }' "$PROMPT_TEMPLATE_FILE" +} + +select_prompt_template() { + local offset="$1" + local total="$2" + local idx + idx=$(( (PROMPT_TEMPLATE_BASE + offset - 1) % total + 1 )) + awk -F'|' -v idx="$idx" ' + NF >= 3 && $1 !~ /^#/ { + n++ + if (n == idx) { + print $1 "\t" $2 "\t" $3 + exit + } + } + ' "$PROMPT_TEMPLATE_FILE" +} + +if [ ! -f "$PROMPT_TEMPLATE_FILE" ]; then + echo "[e2e] prompt template file not found: $PROMPT_TEMPLATE_FILE" >&2 + exit 1 +fi +PROMPT_TEMPLATE_TOTAL=$(prompt_template_count) +if [ "$PROMPT_TEMPLATE_TOTAL" -lt 3 ]; then + echo "[e2e] prompt template file needs at least 3 entries: $PROMPT_TEMPLATE_FILE" >&2 + exit 1 +fi +PROMPT_TEMPLATE_BASE="${IOP_E2E_PROMPT_INDEX:-$((RANDOM % PROMPT_TEMPLATE_TOTAL + 1))}" +IFS=$'\t' read -r FIRST_TEMPLATE FIRST_EXPECTED FIRST_PROMPT < "$MOCK_CLI" #!/usr/bin/env sh while IFS= read -r line; do - case "$line" in - *iop_smoke_alpha*) printf 'IOP_SMOKE_ALPHA\nIOP_SMOKE_ALPHA_TAIL\n' ;; - *iop_smoke_beta*) printf 'IOP_SMOKE_BETA\nIOP_SMOKE_BETA_TAIL\n' ;; - *iop_smoke_bg*) printf 'IOP_SMOKE_BG\nIOP_SMOKE_BG_TAIL\n' ;; - *) printf 'IOP_SMOKE_UNKNOWN\n' ;; - esac + token=$(printf '%s\n' "$line" | sed -nE 's/.*(IOP_E2E_[A-Z0-9_]+).*/\1/p' | head -n 1) + if [ -n "$token" ]; then + printf '%s\n%s_TAIL\n' "$token" "$token" + else + printf 'IOP_E2E_UNKNOWN\n' + fi done EOF chmod +x "$MOCK_CLI" @@ -306,11 +372,11 @@ wait_for_edge_text_since "session_id = default" "$LAST_CMD_START_LINE" "/transpo wait_for_edge_text_since "target = ${TARGET}" "$LAST_CMD_START_LINE" "/transport target" send_cmd "$FIRST_PROMPT" wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 1 start" "$RUN_TIMEOUT" -wait_for_edge_text_since "$FIRST_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 1 response" "$RUN_TIMEOUT" +wait_for_edge_pattern_since "\\[node0-msg\\].*${FIRST_EXPECTED}" "$LAST_CMD_START_LINE" "foreground run 1 node message" "$RUN_TIMEOUT" wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 1 completion" "$RUN_TIMEOUT" send_cmd "$SECOND_PROMPT" wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=" "$LAST_CMD_START_LINE" "foreground run 2 start" "$RUN_TIMEOUT" -wait_for_edge_text_since "$SECOND_EXPECTED" "$LAST_CMD_START_LINE" "foreground run 2 response" "$RUN_TIMEOUT" +wait_for_edge_pattern_since "\\[node0-msg\\].*${SECOND_EXPECTED}" "$LAST_CMD_START_LINE" "foreground run 2 node message" "$RUN_TIMEOUT" wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "foreground run 2 completion" "$RUN_TIMEOUT" send_cmd "/session session2" wait_for_edge_pattern_since "session .*session2" "$LAST_CMD_START_LINE" "/session acknowledgement" @@ -318,7 +384,7 @@ send_cmd "/background on" wait_for_edge_pattern_since "background .*on" "$LAST_CMD_START_LINE" "/background on acknowledgement" send_cmd "$BACKGROUND_PROMPT" wait_for_edge_pattern_since "\\[node0-evt\\] start run_id=.*session=session2.*background=true" "$LAST_CMD_START_LINE" "background run start" "$RUN_TIMEOUT" -wait_for_edge_text_since "$BACKGROUND_EXPECTED" "$LAST_CMD_START_LINE" "background run response" "$RUN_TIMEOUT" +wait_for_edge_pattern_since "\\[node0-msg\\].*${BACKGROUND_EXPECTED}" "$LAST_CMD_START_LINE" "background run node message" "$RUN_TIMEOUT" wait_for_edge_pattern_since "\\[node0-evt\\] complete run_id=" "$LAST_CMD_START_LINE" "background run completion" "$RUN_TIMEOUT" send_cmd "/background off" wait_for_edge_pattern_since "background .*off" "$LAST_CMD_START_LINE" "/background off acknowledgement" @@ -363,7 +429,7 @@ send_cmd "/exit" exec 3>&- wait $EDGE_PID || true -kill $NODE_PID 2>/dev/null || true +kill_process_tree "$NODE_PID" echo "=== EDGE OUTPUT ===" cat "$EDGE_OUT" @@ -416,64 +482,163 @@ check_fail_markers() { } check_node_messages_relayed_to_edge() { - local node_messages="$TMP_DIR/node_messages" - local edge_messages="$TMP_DIR/edge_messages" - local node_counts="$TMP_DIR/node_message_counts" - local count - local payload - local edge_count + local node_runs="$TMP_DIR/node_runs" + local run_id + local node_messages="$TMP_DIR/node_run_messages" + local edge_messages="$TMP_DIR/edge_run_messages" + local edge_order="$TMP_DIR/edge_run_order" + local edge_start_line + local edge_complete_line + local edge_last_message_line + local edge_late_message_line - # Compare the messages the node printed locally with the messages rendered - # by the edge console. A complete event is not enough if some node output - # never reached the edge. + # A complete event is only valid after every node-rendered message line for + # the same run has reached the edge in the same order and with identical + # payload text. awk ' - /^\[node-message\] / { - in_message = 1 - sub(/^\[node-message\] /, "") - if ($0 != "") { - print - } - next + /^\[node-event\] start run_id=/ { + run = $0 + sub(/^\[node-event\] start run_id=/, "", run) + sub(/[[:space:]].*$/, "", run) + print run } - in_message && /^\[/ { - in_message = 0 - } - in_message && /^\{/ { - in_message = 0 - } - in_message && $0 != "" { - print - } - ' "$NODE_OUT" > "$node_messages" + ' "$NODE_OUT" | sort -u > "$node_runs" - awk ' - /^\[[^]]+-msg\] / { - sub(/^\[[^]]+-msg\] /, "") - if ($0 != "") { - print - } - } - ' "$EDGE_OUT" > "$edge_messages" - - if [ ! -s "$node_messages" ]; then - echo "[e2e] FAIL: no node-generated message payloads found in node output" + if [ ! -s "$node_runs" ]; then + echo "[e2e] FAIL: no node run ids found in node output" FAIL=1 return fi - sort "$node_messages" | uniq -c > "$node_counts" - while read -r count payload; do - if [ -z "$payload" ]; then + while IFS= read -r run_id; do + if [ -z "$run_id" ]; then continue fi - edge_count=$(grep -Fxc -- "$payload" "$edge_messages" || true) - if [ "$edge_count" -lt "$count" ]; then - echo "[e2e] FAIL: node message payload was not fully relayed to edge" - echo "[e2e] missing-or-short payload: $payload" - echo "[e2e] node_count=$count edge_count=$edge_count" + + awk -v want="$run_id" ' + /^\[node-event\] start run_id=/ { + run = $0 + sub(/^\[node-event\] start run_id=/, "", run) + sub(/[[:space:]].*$/, "", run) + in_message = 0 + next + } + /^\[node-event\] complete run_id=/ { + in_message = 0 + next + } + /^\[node-message\] / { + in_message = (run == want) + if (run == want) { + line = $0 + sub(/^\[node-message\] /, "", line) + if (line != "") { + print line + } + } + next + } + in_message && /^\[/ { + in_message = 0 + } + in_message && /^\{/ { + in_message = 0 + } + in_message && $0 != "" { + print + } + ' "$NODE_OUT" > "$node_messages" + + awk -v want="$run_id" ' + $0 ~ /\[[^]]+-evt\] start run_id=/ { + if (after_want_complete) { + after_want_complete = 0 + } + run = $0 + sub(/^.*\[[^]]+-evt\] start run_id=/, "", run) + sub(/[[:space:]].*$/, "", run) + current = run + if (run == want) { + print NR "\tSTART\t" + } + next + } + $0 ~ /\[[^]]+-evt\] complete run_id=/ { + run = $0 + sub(/^.*\[[^]]+-evt\] complete run_id=/, "", run) + sub(/[[:space:]].*$/, "", run) + if (run == want) { + print NR "\tCOMPLETE\t" + after_want_complete = 1 + } + if (current == run) { + current = "" + } + next + } + $0 ~ /\[[^]]+-msg\] / { + if (current == want) { + line = $0 + sub(/^.*\[[^]]+-msg\] /, "", line) + if (line != "") { + print NR "\tMSG\t" line + } + } else if (after_want_complete) { + line = $0 + sub(/^.*\[[^]]+-msg\] /, "", line) + print NR "\tLATE_MSG\t" line + } + } + ' "$EDGE_OUT" > "$edge_order" + + awk -F '\t' '$2 == "MSG" { print $3 }' "$edge_order" > "$edge_messages" + edge_start_line=$(awk -F '\t' '$2 == "START" { print $1; exit }' "$edge_order") + edge_complete_line=$(awk -F '\t' '$2 == "COMPLETE" { print $1; exit }' "$edge_order") + edge_last_message_line=$(awk -F '\t' '$2 == "MSG" { line = $1 } END { print line }' "$edge_order") + edge_late_message_line=$(awk -F '\t' '$2 == "LATE_MSG" { print $1; exit }' "$edge_order") + + if [ ! -s "$node_messages" ]; then + echo "[e2e] FAIL: no node-generated message payloads found for run $run_id" + FAIL=1 + continue + fi + + if [ -z "$edge_start_line" ]; then + echo "[e2e] FAIL: edge start event not found for run $run_id" + FAIL=1 + continue + fi + + if [ ! -s "$edge_messages" ]; then + echo "[e2e] FAIL: no edge-rendered message payloads found for run $run_id" + FAIL=1 + continue + fi + + if [ -z "$edge_complete_line" ]; then + echo "[e2e] FAIL: edge completion not found for run $run_id" + FAIL=1 + continue + fi + + if [ -z "$edge_last_message_line" ] || [ "$edge_complete_line" -le "$edge_last_message_line" ]; then + echo "[e2e] FAIL: edge complete arrived before all rendered messages for run $run_id" + echo "[e2e] complete_line=$edge_complete_line last_message_line=${edge_last_message_line:-}" FAIL=1 fi - done < "$node_counts" + + if [ -n "$edge_late_message_line" ]; then + echo "[e2e] FAIL: edge rendered a message after complete for run $run_id" + awk -F '\t' '$2 == "LATE_MSG" { print "[e2e] late_message_line=" $1 " payload=" $3 }' "$edge_order" + FAIL=1 + fi + + if ! diff -u "$node_messages" "$edge_messages" > "$TMP_DIR/message_diff"; then + echo "[e2e] FAIL: node message payloads differ from edge-rendered payloads for run $run_id" + cat "$TMP_DIR/message_diff" + FAIL=1 + fi + done < "$node_runs" } check_grep "test-node" "$EDGE_OUT" "node registration not found" @@ -483,13 +648,13 @@ if [ "$(grep -c "\\[node0-evt\\] complete run_id=" "$EDGE_OUT" || true)" -lt 3 ] echo "[e2e] FAIL: expected at least 3 completed runs (foreground x2 + background x1)" FAIL=1 fi -check_grep "$FIRST_EXPECTED" "$EDGE_OUT" "foreground run 1 response not found" -check_grep "$SECOND_EXPECTED" "$EDGE_OUT" "foreground run 2 response not found" -check_grep "$BACKGROUND_EXPECTED" "$EDGE_OUT" "background run response not found" +check_grep "\\[node0-msg\\].*${FIRST_EXPECTED}" "$EDGE_OUT" "foreground run 1 node message not found" +check_grep "\\[node0-msg\\].*${SECOND_EXPECTED}" "$EDGE_OUT" "foreground run 2 node message not found" +check_grep "\\[node0-msg\\].*${BACKGROUND_EXPECTED}" "$EDGE_OUT" "background run node message not found" if [ "$EXPECT_TAIL_MESSAGES" -eq 1 ]; then - check_grep "$FIRST_TAIL_EXPECTED" "$EDGE_OUT" "foreground run 1 tail response not found" - check_grep "$SECOND_TAIL_EXPECTED" "$EDGE_OUT" "foreground run 2 tail response not found" - check_grep "$BACKGROUND_TAIL_EXPECTED" "$EDGE_OUT" "background run tail response not found" + check_grep "\\[node0-msg\\].*${FIRST_TAIL_EXPECTED}" "$EDGE_OUT" "foreground run 1 tail node message not found" + check_grep "\\[node0-msg\\].*${SECOND_TAIL_EXPECTED}" "$EDGE_OUT" "foreground run 2 tail node message not found" + check_grep "\\[node0-msg\\].*${BACKGROUND_TAIL_EXPECTED}" "$EDGE_OUT" "background run tail node message not found" fi check_no_grep "\\[node0-msg\\] " "$EDGE_OUT" "empty node message found" check_node_messages_relayed_to_edge diff --git a/scripts/fixtures/user-e2e-prompts.tsv b/scripts/fixtures/user-e2e-prompts.tsv new file mode 100644 index 0000000..970a87b --- /dev/null +++ b/scripts/fixtures/user-e2e-prompts.tsv @@ -0,0 +1,11 @@ +# slug|expected|prompt +hello-basic|IOP_E2E_HELLO_BASIC|짧은 인사 확인입니다. 답변은 정확히 IOP_E2E_HELLO_BASIC 하나만 출력하세요. +hello-formal|IOP_E2E_HELLO_FORMAL|안녕하세요에 대한 응답 확인입니다. 다른 말 없이 IOP_E2E_HELLO_FORMAL 만 답하세요. +thanks-short|IOP_E2E_THANKS_SHORT|고맙다는 말에 대한 짧은 확인입니다. 출력은 IOP_E2E_THANKS_SHORT 만 사용하세요. +thanks-formal|IOP_E2E_THANKS_FORMAL|감사합니다라는 상황입니다. 답변은 정확히 IOP_E2E_THANKS_FORMAL 하나만 쓰세요. +ping-basic|IOP_E2E_PING_BASIC|간단한 ping 확인입니다. IOP_E2E_PING_BASIC 만 출력하세요. +status-ok|IOP_E2E_STATUS_OK|상태가 정상인지 짧게 확인합니다. 답은 IOP_E2E_STATUS_OK 하나만 출력하세요. +ack-short|IOP_E2E_ACK_SHORT|확인했다는 짧은 응답 테스트입니다. IOP_E2E_ACK_SHORT 만 답하세요. +ready-short|IOP_E2E_READY_SHORT|준비되었는지 묻는 짧은 테스트입니다. IOP_E2E_READY_SHORT 만 출력하세요. +bye-short|IOP_E2E_BYE_SHORT|짧은 마무리 인사 테스트입니다. 답변은 IOP_E2E_BYE_SHORT 만 쓰세요. +yes-short|IOP_E2E_YES_SHORT|간단한 긍정 응답 확인입니다. 정확히 IOP_E2E_YES_SHORT 만 출력하세요.