iop/apps/agent/internal/taskloop/cutover_test.go
toki a2becd222a chore(iop): 런타임 변경과 마일스톤 아카이브를 반영한다
완료된 IOP Agent CLI Runtime의 상태·스펙·계약·작업 evidence를 아카이브 경로로 동기화하고 현재 런타임 검증 변경을 원격에 공유한다.
2026-07-31 21:57:38 +09:00

706 lines
21 KiB
Go

package taskloop
import (
"bufio"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
)
const dispatcherPlanValidatorCommand = "python3 agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py --workspace <workspace> --validate-plan <candidate-plan>"
var cutoverStaticOwnershipDocuments = []string{
"agent-ops/skills/common/plan/SKILL.md",
"agent-ops/skills/common/code-review/SKILL.md",
"agent-ops/rules/project/domain/testing/rules.md",
"agent-ops/rules/project/rules.md",
}
var cutoverProductionRoots = []string{
"agent-ops/skills/common",
"agent-ops/skills/project",
"agent-ops/rules",
}
func TestClassifyExactValidationDocumentationAcceptsDeclaredForms(t *testing.T) {
tests := []struct {
name string
line string
expectedDeclared string
expectedText string
}{
{
name: "raw_candidate",
line: exactValidationCommandCandidates[0],
expectedDeclared: exactValidationCommandCandidates[0],
expectedText: "",
},
{
name: "raw_written",
line: exactValidationCommandCandidates[1],
expectedDeclared: exactValidationCommandCandidates[1],
expectedText: "",
},
{
name: "single_backtick_candidate",
line: "`" + exactValidationCommandCandidates[0] + "`",
expectedDeclared: exactValidationCommandCandidates[0],
expectedText: "",
},
{
name: "single_backtick_written",
line: "`" + exactValidationCommandCandidates[1] + "`",
expectedDeclared: exactValidationCommandCandidates[1],
expectedText: "",
},
{
name: "list_item_hyphen_candidate",
line: "- Validate with `" + exactValidationCommandCandidates[0] + "` before write.",
expectedDeclared: exactValidationCommandCandidates[0],
expectedText: "- Validate with before write.",
},
{
name: "list_item_asterisk_written",
line: "* Re-run `" + exactValidationCommandCandidates[1] + "` to confirm.",
expectedDeclared: exactValidationCommandCandidates[1],
expectedText: "* Re-run to confirm.",
},
{
name: "list_item_numbered_candidate",
line: "1. Run `" + exactValidationCommandCandidates[0] + "` now.",
expectedDeclared: exactValidationCommandCandidates[0],
expectedText: "1. Run now.",
},
{
name: "list_item_with_extra_code_span",
line: "- Check `prepared_plan` then run `" + exactValidationCommandCandidates[0] + "` to validate.",
expectedDeclared: exactValidationCommandCandidates[0],
expectedText: "- Check `prepared_plan` then run to validate.",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
text, declared := classifyExactValidationDocumentation(tt.line)
if len(declared) != 1 || declared[0] != tt.expectedDeclared {
t.Fatalf("classifyExactValidationDocumentation declared = %v, expected [%s]", declared, tt.expectedDeclared)
}
if text != tt.expectedText {
t.Fatalf("classifyExactValidationDocumentation text = %q, expected %q", text, tt.expectedText)
}
})
}
}
func TestCutoverProductionOwnershipHasNoReferenceCallerOrStaticRouteTable(t *testing.T) {
root := repoRoot(t)
if err := validateCutoverProductionOwnership(root); err != nil {
t.Fatal(err)
}
}
func TestCutoverProductionOwnershipRejectsInjectedPythonCaller(t *testing.T) {
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/other/SKILL.md"
path := filepath.Join(root, relative)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("python3 agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py --validate-plan candidate\n"), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), "matches") || !strings.Contains(err.Error(), ":1") {
t.Fatalf("validateCutoverProductionOwnership error = %v", err)
}
}
func TestCutoverProductionOwnershipRejectsUnexpectedCallerInAllowedDocument(t *testing.T) {
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/plan/SKILL.md"
path := filepath.Join(root, relative)
contents := dispatcherPlanValidatorCommand + "\npython3 agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py --dry-run\n"
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), relative+":2") || !strings.Contains(err.Error(), "matches") {
t.Fatalf("validateCutoverProductionOwnership error = %v", err)
}
}
func TestCutoverProductionOwnershipRejectsNonExactValidationVariants(t *testing.T) {
tests := []struct {
name string
contents string
}{
{
name: "env_prefix",
contents: "env " + dispatcherPlanValidatorCommand + "\n",
},
{
name: "command_substitution",
contents: "$(" + dispatcherPlanValidatorCommand + ")\n",
},
{
name: "backtick_command_substitution",
contents: "echo `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "output_redirection",
contents: dispatcherPlanValidatorCommand + " > /tmp/validation.out\n",
},
{
name: "positional_suffix",
contents: dispatcherPlanValidatorCommand + " unexpected\n",
},
{
name: "quoted_flag_suffix",
contents: dispatcherPlanValidatorCommand + " --dry-run\n",
},
{
name: "shell_operator_prefix",
contents: "true && " + dispatcherPlanValidatorCommand + "\n",
},
{
name: "shell_wrapper_prefix",
contents: `bash -c "` + dispatcherPlanValidatorCommand + `"` + "\n",
},
{
name: "multiple_occurrences",
contents: dispatcherPlanValidatorCommand + " " + dispatcherPlanValidatorCommand + "\n",
},
{
name: "repeated_inline_spans",
contents: "- check `" + dispatcherPlanValidatorCommand + "` and `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "malformed_code_span",
contents: "`" + dispatcherPlanValidatorCommand + "\n",
},
{
name: "shell_semicolon_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "`; echo done\n",
},
{
name: "command_substitution_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` $(echo done)\n",
},
{
name: "double_backtick_span",
contents: "- check ``" + dispatcherPlanValidatorCommand + "``\n",
},
{
name: "non_adjacent_semicolon_prefix",
contents: "- preface ; then `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_semicolon_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` then ; echo done\n",
},
{
name: "non_adjacent_semicolon_unlisted_prefix",
contents: "- Validate; please run `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_semicolon_unlisted_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` when ready; printf done\n",
},
{
name: "non_adjacent_ampersand_prefix",
contents: "- preface & then `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_ampersand_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` then & echo done\n",
},
{
name: "non_adjacent_pipe_prefix",
contents: "- preface | then `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_pipe_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` then | grep pattern\n",
},
{
name: "non_adjacent_less_than_prefix",
contents: "- preface < input.txt then `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_less_than_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` then < input.txt\n",
},
{
name: "non_adjacent_greater_than_prefix",
contents: "- preface > output.txt then `" + dispatcherPlanValidatorCommand + "`\n",
},
{
name: "non_adjacent_greater_than_suffix",
contents: "- check `" + dispatcherPlanValidatorCommand + "` then > output.txt\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
line := strings.TrimSuffix(tt.contents, "\n")
directText, declared := classifyExactValidationDocumentation(line)
if len(declared) != 0 || directText != line {
t.Fatalf("classifyExactValidationDocumentation(%q) = (%q, %v), expected (%q, nil)", line, directText, declared, line)
}
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/plan/SKILL.md"
path := filepath.Join(root, relative)
contents := dispatcherPlanValidatorCommand + "\n" + tt.contents
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), relative+":2") || !strings.Contains(err.Error(), "matches") {
t.Fatalf("validateCutoverProductionOwnership error = %v for case %s", err, tt.name)
}
})
}
}
func TestCutoverProductionOwnershipRejectsMissingDeclaredValidationCommand(t *testing.T) {
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/plan/SKILL.md"
path := filepath.Join(root, relative)
if err := os.WriteFile(path, []byte("# Empty plan skill with no declared validation command\n"), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), relative) || !strings.Contains(err.Error(), "lacks") {
t.Fatalf("validateCutoverProductionOwnership error = %v", err)
}
}
func TestCutoverProductionOwnershipRejectsSuffixedValidationCommand(t *testing.T) {
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/plan/SKILL.md"
path := filepath.Join(root, relative)
contents := dispatcherPlanValidatorCommand + " --dry-run\n"
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), relative+":1") || !strings.Contains(err.Error(), "matches") {
t.Fatalf("validateCutoverProductionOwnership error = %v", err)
}
}
func TestCutoverProductionOwnershipRejectsWrappedPythonCaller(t *testing.T) {
root := createCutoverOwnershipFixture(t)
relative := "agent-ops/skills/common/plan/SKILL.md"
path := filepath.Join(root, relative)
contents := `bash -c "` + dispatcherPlanValidatorCommand + `"` + "\n"
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
err := validateCutoverProductionOwnership(root)
if err == nil || !strings.Contains(err.Error(), relative+":1") || !strings.Contains(err.Error(), "matches") {
t.Fatalf("validateCutoverProductionOwnership error = %v", err)
}
}
func TestCutoverNodeRemainsSharedRuntimeConsumer(t *testing.T) {
root := repoRoot(t)
patterns := []*regexp.Regexp{
regexp.MustCompile(`agenttask\.NewManager\(`),
regexp.MustCompile(`taskloop\.New\(`),
regexp.MustCompile(`NewProvider\(`),
}
err := filepath.WalkDir(filepath.Join(root, "apps", "node"), func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") {
return nil
}
assertNoPatterns(t, path, patterns)
return nil
})
if err != nil {
t.Fatal(err)
}
}
func assertNoPatterns(t *testing.T, path string, patterns []*regexp.Regexp) {
t.Helper()
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
line := 0
for scanner.Scan() {
line++
for _, pattern := range patterns {
if pattern.MatchString(scanner.Text()) {
t.Fatalf("cutover ownership violation %s:%d matches %q", path, line, pattern.String())
}
}
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
}
func validateCutoverProductionOwnership(root string) error {
callerPatterns := []*regexp.Regexp{
regexp.MustCompile(`\bpython3?\s+.*dispatch\.py\b`),
regexp.MustCompile(`\bpython3?\s+.*orchestrate-agent-task-loop`),
}
staticOwnershipPatterns := []*regexp.Regexp{
regexp.MustCompile(`gpt-5\.6`),
regexp.MustCompile(`claude-opus`),
regexp.MustCompile(`Gemini`),
regexp.MustCompile(`ornith`),
regexp.MustCompile(`laguna`),
}
documents, err := discoverCutoverProductionDocuments(root)
if err != nil {
return err
}
for _, relative := range documents {
path := filepath.Join(root, relative)
isValidationDoc := isDeclaredDispatcherValidationDocument(relative)
declared, err := noDispatcherCallerViolation(path, callerPatterns, isValidationDoc)
if err != nil {
return err
}
if isValidationDoc {
if relative == "agent-ops/skills/common/plan/SKILL.md" {
hasCandidate := false
for _, cmd := range declared {
if cmd == exactValidationCommandCandidates[0] {
hasCandidate = true
break
}
}
if !hasCandidate {
return fmt.Errorf("cutover ownership violation %s lacks declared candidate validation command", path)
}
} else if relative == "agent-ops/skills/common/code-review/SKILL.md" {
hasWritten := false
for _, cmd := range declared {
if cmd == exactValidationCommandCandidates[1] {
hasWritten = true
break
}
}
if !hasWritten {
return fmt.Errorf("cutover ownership violation %s lacks declared written validation command", path)
}
} else if len(declared) == 0 {
return fmt.Errorf("cutover ownership violation %s lacks declared validation command", path)
}
}
}
for _, relative := range cutoverStaticOwnershipDocuments {
path := filepath.Join(root, relative)
if err := noPatternViolation(path, staticOwnershipPatterns); err != nil {
return err
}
}
return nil
}
func discoverCutoverProductionDocuments(root string) ([]string, error) {
var documents []string
for _, relativeRoot := range cutoverProductionRoots {
directory := filepath.Join(root, filepath.FromSlash(relativeRoot))
err := filepath.WalkDir(directory, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
relative, err := filepath.Rel(root, path)
if err != nil {
return err
}
relative = filepath.ToSlash(relative)
if entry.Name() == "tests" || strings.HasPrefix(entry.Name(), ".") || relative == "agent-ops/skills/project/orchestrate-agent-task-loop/scripts" {
return filepath.SkipDir
}
return nil
}
if !entry.Type().IsRegular() {
return nil
}
relative, err := filepath.Rel(root, path)
if err != nil {
return err
}
relative = filepath.ToSlash(relative)
if isDeclaredDispatcherOwner(relative) {
return nil
}
documents = append(documents, relative)
return nil
})
if err != nil {
return nil, err
}
}
sort.Strings(documents)
return documents, nil
}
func createCutoverOwnershipFixture(t *testing.T) string {
t.Helper()
root := t.TempDir()
for _, relativeRoot := range cutoverProductionRoots {
if err := os.MkdirAll(filepath.Join(root, relativeRoot), 0700); err != nil {
t.Fatal(err)
}
}
for _, relative := range cutoverStaticOwnershipDocuments {
contents := ""
if relative == "agent-ops/skills/common/plan/SKILL.md" {
contents = exactValidationCommandCandidates[0] + "\n"
} else if relative == "agent-ops/skills/common/code-review/SKILL.md" {
contents = exactValidationCommandCandidates[1] + "\n"
}
path := filepath.Join(root, relative)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}
return root
}
func isDeclaredDispatcherOwner(relative string) bool {
return relative == "agent-ops/skills/project/orchestrate-agent-task-loop/SKILL.md"
}
func isDeclaredDispatcherValidationDocument(relative string) bool {
return relative == "agent-ops/skills/common/plan/SKILL.md" ||
relative == "agent-ops/skills/common/code-review/SKILL.md"
}
var exactValidationCommandCandidates = []string{
"python3 agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py --workspace <workspace> --validate-plan <candidate-plan>",
"python3 agent-ops/skills/project/orchestrate-agent-task-loop/scripts/dispatch.py --workspace <workspace> --validate-plan <written-plan>",
}
type markdownCodeSpan struct {
delimiterLen int
content string
rawSpan string
prefixProse string
suffixProse string
runStart int
closeEnd int
}
func parseMarkdownCodeSpans(line string) ([]markdownCodeSpan, bool) {
var spans []markdownCodeSpan
pos := 0
lastEnd := 0
for pos < len(line) {
if line[pos] != '`' {
pos++
continue
}
runStart := pos
for pos < len(line) && line[pos] == '`' {
pos++
}
d := pos - runStart
closeStart := -1
searchPos := pos
for searchPos < len(line) {
idx := strings.Index(line[searchPos:], strings.Repeat("`", d))
if idx == -1 {
break
}
candStart := searchPos + idx
candEnd := candStart + d
if candStart > 0 && line[candStart-1] == '`' {
searchPos = candEnd
continue
}
if candEnd < len(line) && line[candEnd] == '`' {
searchPos = candEnd
continue
}
closeStart = candStart
break
}
if closeStart == -1 {
return nil, false
}
closeEnd := closeStart + d
prefixProse := line[lastEnd:runStart]
content := line[runStart+d : closeStart]
rawSpan := line[runStart:closeEnd]
spans = append(spans, markdownCodeSpan{
delimiterLen: d,
content: content,
rawSpan: rawSpan,
prefixProse: prefixProse,
runStart: runStart,
closeEnd: closeEnd,
})
lastEnd = closeEnd
pos = closeEnd
}
if len(spans) == 0 {
return []markdownCodeSpan{{suffixProse: line}}, true
}
spans[len(spans)-1].suffixProse = line[lastEnd:]
return spans, true
}
func validationProseSegments(spans []markdownCodeSpan) []string {
prose := make([]string, 0, len(spans)+1)
for _, span := range spans {
prose = append(prose, span.prefixProse)
}
if len(spans) > 0 {
prose = append(prose, spans[len(spans)-1].suffixProse)
}
return prose
}
var unsafeValidationProseControlWords = regexp.MustCompile(`\b(echo|eval|exec|bash|sh|env)\b`)
func containsUnsafeValidationProse(prose string) bool {
if strings.Contains(prose, "$(") ||
strings.Contains(prose, "&&") ||
strings.Contains(prose, "||") ||
strings.ContainsAny(prose, ";><|&") ||
unsafeValidationProseControlWords.MatchString(prose) {
return true
}
return false
}
func classifyExactValidationDocumentation(line string) (string, []string) {
var matchedCandidates []string
totalOccurrences := 0
for _, exactCmd := range exactValidationCommandCandidates {
count := strings.Count(line, exactCmd)
if count > 0 {
totalOccurrences += count
matchedCandidates = append(matchedCandidates, exactCmd)
}
}
if totalOccurrences != 1 {
return line, nil
}
exactCmd := matchedCandidates[0]
trimmed := strings.TrimSpace(line)
if trimmed == exactCmd || trimmed == "`"+exactCmd+"`" {
return "", []string{exactCmd}
}
isListItem := strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") || regexp.MustCompile(`^\d+\.\s`).MatchString(trimmed)
if !isListItem {
return line, nil
}
spans, ok := parseMarkdownCodeSpans(line)
if !ok {
return line, nil
}
exactCandidateSpanCount := 0
exactCandidateIsSingleBacktick := false
var exactSpanIndex int
for i, span := range spans {
if span.content == exactCmd {
exactCandidateSpanCount++
if span.delimiterLen == 1 {
exactCandidateIsSingleBacktick = true
exactSpanIndex = i
}
}
}
if exactCandidateSpanCount != 1 || !exactCandidateIsSingleBacktick {
return line, nil
}
for _, prose := range validationProseSegments(spans) {
if containsUnsafeValidationProse(prose) {
return line, nil
}
}
var sb strings.Builder
for i, span := range spans {
sb.WriteString(span.prefixProse)
if i != exactSpanIndex {
sb.WriteString(span.rawSpan)
}
}
sb.WriteString(spans[len(spans)-1].suffixProse)
outsideText := sb.String()
return outsideText, []string{exactCmd}
}
func noDispatcherCallerViolation(path string, patterns []*regexp.Regexp, allowDeclaredValidationCommand bool) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
line := 0
var declaredCommands []string
for scanner.Scan() {
line++
content := scanner.Text()
if allowDeclaredValidationCommand {
var declared []string
content, declared = classifyExactValidationDocumentation(content)
declaredCommands = append(declaredCommands, declared...)
}
for _, pattern := range patterns {
if pattern.MatchString(content) {
return nil, fmt.Errorf("cutover ownership violation %s:%d matches %q", path, line, pattern.String())
}
}
}
return declaredCommands, scanner.Err()
}
func noPatternViolation(path string, patterns []*regexp.Regexp) error {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
line := 0
for scanner.Scan() {
line++
for _, pattern := range patterns {
if pattern.MatchString(scanner.Text()) {
return fmt.Errorf("cutover ownership violation %s:%d matches %q", path, line, pattern.String())
}
}
}
return scanner.Err()
}