package operator import ( "bufio" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" ) // requiredScenarios are the six headless validation scenarios the milestone // handoff table requires evidence for. The handoff matrix and the expected // output fixtures must cover all of them so the Flutter Operator Console MVP // can validate each workflow before building any screen. var requiredScenarios = []string{ "api_connection_smoke", "market_data_status_query", "us_market_data_status_query", "backtest_run_request", "backtest_run_polling", "backtest_result_summary", "backtest_matrix_success", "backtest_matrix_failed", "backtest_matrix_timeout", "backtest_matrix_mismatch", "invalid_request_matrix", "kis_daily_import_smoke", "minute_import_rejected", "paper_trading_state", "paper_order_lifecycle", "live_order_lifecycle", "scheduler_refresh_status", } func handoffMatrixPath() string { return filepath.Join("..", "..", "testdata", "operator", "headless_validation.md") } func expectedFixturePath(scenario string) string { return filepath.Join("..", "..", "testdata", "operator", "expected", scenario+".jsonl") } // TestHeadlessValidationHandoffCoversRequiredScenarios fails if the handoff // matrix drops any required scenario, which would leave the Flutter MVP without // command/field evidence for that workflow. func TestHeadlessValidationHandoffCoversRequiredScenarios(t *testing.T) { data, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } doc := string(data) for _, s := range requiredScenarios { if !strings.Contains(doc, "`"+s+"`") { t.Errorf("handoff matrix missing scenario %q", s) } } } // TestExpectedOutputFixturesAreValidJSONLines checks that every expected output // fixture parses as JSON lines and that each record carries the machine-readable // keys the handoff contract promises: scenario/status on every line, action on // step lines, plus at least one step and one summary line. func TestExpectedOutputFixturesAreValidJSONLines(t *testing.T) { for _, s := range requiredScenarios { path := expectedFixturePath(s) f, err := os.Open(path) if err != nil { t.Errorf("open expected fixture %q: %v", path, err) continue } scanner := bufio.NewScanner(f) lineNo := 0 sawStep := false sawSummary := false for scanner.Scan() { lineNo++ line := strings.TrimSpace(scanner.Text()) if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Errorf("%s:%d not valid JSON: %v", path, lineNo, err) continue } if rec["scenario"] != s { t.Errorf("%s:%d scenario = %v, want %q", path, lineNo, rec["scenario"], s) } if _, ok := rec["status"]; !ok { t.Errorf("%s:%d missing status field", path, lineNo) } switch rec["type"] { case "step": sawStep = true if _, ok := rec["action"]; !ok { t.Errorf("%s:%d step line missing action field", path, lineNo) } case "summary": sawSummary = true default: t.Errorf("%s:%d unexpected type %v", path, lineNo, rec["type"]) } } if err := scanner.Err(); err != nil { t.Errorf("scan %q: %v", path, err) } f.Close() if !sawStep { t.Errorf("%s has no step line", path) } if !sawSummary { t.Errorf("%s has no summary line", path) } } } // TestStatusScenarioFixtureImportsBeforeReads guards the command-first status // scenario: its expected fixture must carry an import_daily_bars step line with // instrument_count and bar_count keys before the instrument/bar read lines, so // the handoff evidence shows imported counts rather than a bare status read. func TestStatusScenarioFixtureImportsBeforeReads(t *testing.T) { const scenario = "market_data_status_query" f, err := os.Open(expectedFixturePath(scenario)) if err != nil { t.Fatalf("open expected fixture: %v", err) } defer f.Close() scanner := bufio.NewScanner(f) importLine := -1 instLine := -1 barLine := -1 lineNo := 0 for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } lineNo++ var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatalf("line %d not valid JSON: %v", lineNo, err) } switch rec["action"] { case "import_daily_bars": importLine = lineNo if _, ok := rec["instrument_count"]; !ok { t.Errorf("import step line %d missing instrument_count", lineNo) } if _, ok := rec["bar_count"]; !ok { t.Errorf("import step line %d missing bar_count", lineNo) } case "list_instruments": instLine = lineNo case "list_bars": barLine = lineNo } } if err := scanner.Err(); err != nil { t.Fatalf("scan fixture: %v", err) } if importLine < 0 { t.Fatal("status fixture has no import_daily_bars step line") } if instLine < 0 || barLine < 0 { t.Fatal("status fixture missing instrument/bar read step lines") } if importLine > instLine || importLine > barLine { t.Errorf("import line %d must precede read lines (instruments=%d, bars=%d)", importLine, instLine, barLine) } } func TestUSStatusScenarioFixtureImportsBeforeReads(t *testing.T) { const scenario = "us_market_data_status_query" f, err := os.Open(expectedFixturePath(scenario)) if err != nil { t.Fatalf("open expected fixture: %v", err) } defer f.Close() scanner := bufio.NewScanner(f) importLine := -1 instLine := -1 barLine := -1 lineNo := 0 for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } lineNo++ var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatalf("line %d not valid JSON: %v", lineNo, err) } switch rec["action"] { case "import_daily_bars": importLine = lineNo if _, ok := rec["instrument_count"]; !ok { t.Errorf("import step line %d missing instrument_count", lineNo) } if _, ok := rec["bar_count"]; !ok { t.Errorf("import step line %d missing bar_count", lineNo) } case "list_instruments": instLine = lineNo case "list_bars": barLine = lineNo } } if err := scanner.Err(); err != nil { t.Fatalf("scan fixture: %v", err) } if importLine < 0 { t.Fatal("status fixture has no import_daily_bars step line") } if instLine < 0 || barLine < 0 { t.Fatal("status fixture missing instrument/bar read step lines") } if importLine > instLine || importLine > barLine { t.Errorf("import line %d must precede read lines (instruments=%d, bars=%d)", importLine, instLine, barLine) } } // requiredScenarioKeys are the expected output keys each required scenario's // JSONL fixture must carry. Kept as a fallback so existing tests still pass, // but the definitive source of truth is now the handoff matrix itself via // TestHandoffExpectedFixturesWithMatrixPromisedKeys. var requiredScenarioKeys = map[string][]string{ "api_connection_smoke": {"scenario", "status", "type", "action"}, "backtest_result_summary": {"run_id", "run_status", "starting_cash", "ending_equity", "total_return", "trade_count"}, "backtest_run_polling": {"run_id", "run_status", "provider", "bar_count", "instrument_count"}, "backtest_run_request": {"run_id", "run_status", "bar_count", "instrument_count", "provider"}, "backtest_matrix_success": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"}, "backtest_matrix_failed": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"}, "backtest_matrix_timeout": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"}, "backtest_matrix_mismatch": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"}, "invalid_request_matrix": {"scenario", "status", "type", "action", "error_code", "error_message"}, "kis_daily_import_smoke": {"provider", "instrument_count", "bar_count"}, "minute_import_rejected": {"scenario", "status", "type", "action", "error_code", "error_message"}, "market_data_status_query": {"provider", "instrument_count", "bar_count", "count", "universe", "freshness_status", "missing_count", "latest_yyyymmdd", "gap_status", "gap_count", "duplicate_count", "provider_delay_days"}, "us_market_data_status_query": {"provider", "instrument_count", "bar_count", "count", "universe", "freshness_status", "missing_count", "latest_yyyymmdd", "gap_status", "gap_count", "duplicate_count", "provider_delay_days"}, "paper_trading_state": {"account_id", "run_id", "run_status", "cash", "equity_point_count", "fill_count", "latest_equity", "position_count", "risk"}, "paper_order_lifecycle": {"scenario", "status", "type", "action", "account_id", "run_id", "run_status", "cash", "equity_point_count", "fill_count", "latest_equity", "position_count", "risk", "order_id", "order_status", "fill_price"}, "live_order_lifecycle": {"scenario", "status", "type", "action", "live_order_id", "live_order_status"}, "scheduler_refresh_status": {"scenario", "status", "type", "action", "schedule", "scheduler_status", "last_success_unix_ms", "last_error", "next_run_unix_ms", "imported_bar_count", "missing_count", "gap_count", "duplicate_count", "provider_delay_days"}, } // TestHandoffMatrixDocumentsCommandFirstColumns asserts that the handoff // matrix has been updated from the legacy wireframe-only columns to the // command-first columns introduced in the current milestone. The matrix must // contain explicit section text and column headers for expected output keys, // repeatable operation, UI candidate, and UI defer reason. func TestHandoffMatrixDocumentsCommandFirstColumns(t *testing.T) { data, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } doc := string(data) requiredHeaders := []string{ "Expected output keys", "Repeatable operation", "UI candidate", "UI defer reason", } for _, header := range requiredHeaders { if !strings.Contains(doc, header) { t.Errorf("handoff matrix missing required column header %q", header) } } // Also verify the new section is present (not just the old header text) if !strings.Contains(doc, "command-first workflow handoff matrix") { t.Error("handoff matrix missing 'command-first workflow handoff matrix' section") } } // TestHandoffExpectedFixturesContainPromisedKeys asserts that every required scenario's // JSONL fixture contains at least one line with each key promised in the // handoff matrix. Kept as a secondary guard using requiredScenarioKeys. func TestHandoffExpectedFixturesContainPromisedKeys(t *testing.T) { // Ensure the requiredScenarios slice matches our expectedScenarioKeys map if len(requiredScenarios) == 0 { t.Fatal("requiredScenarios must not be empty") } for _, s := range requiredScenarios { keys, ok := requiredScenarioKeys[s] if !ok { t.Fatalf("required scenario %q has no promised keys defined in requiredScenarioKeys", s) } if len(keys) == 0 { t.Fatalf("required scenario %q has an empty promised keys list in requiredScenarioKeys", s) } path := expectedFixturePath(s) f, err := os.Open(path) if err != nil { t.Fatalf("open expected fixture for %q: %v", s, err) } found := make(map[string]bool) scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Errorf("%s not valid JSON: %v", path, err) continue } for k := range rec { if found[k] { continue } for _, want := range keys { if k == want { found[k] = true } } } } if err := scanner.Err(); err != nil { t.Fatalf("scan %q: %v", path, err) } f.Close() for _, want := range keys { if !found[want] { t.Errorf("%s (%q) missing promised key %q", path, s, want) } } } } // TestHandoffExpectedFixturesWithMatrixPromisedKeys reads the command-first handoff // matrix from the handoff doc, finds the "Expected output keys" cell for each // required scenario row, extracts backtick-wrapped key names, and verifies // each promised key appears at least once in that scenario's expected JSONL // fixture. The test fails loudly when: // - a required scenario row is missing from the matrix, // - the matrix promises zero keys (empty Expected output keys cell), // - a promised key is absent from the expected fixture. // // This is the definitive matrix-driven key contract test introduced by // local-G06 so the test source of truth is the handoff doc, not a hardcoded // map. func TestHandoffExpectedFixturesWithMatrixPromisedKeys(t *testing.T) { data, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } lines := strings.Split(string(data), "\n") // Find the table header: first line containing both "Scenario" and "Command" headerLineIdx := -1 for i, line := range lines { if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") { headerLineIdx = i break } } if headerLineIdx < 0 { t.Fatal("matrix header row not found in handoff doc") } // The next line is the separator row (|---|---|...), skip it. // Data rows start at headerLineIdx+2. // Build a column-index map from the header row. headerCells := splitTableRow(lines[headerLineIdx]) colIndex := mapCol(headerCells, map[string]string{ "scenario": "Scenario", "expectedOutputKeys": "Expected output keys", }) // Iterate the matrix data rows (start after the separator row after header). scenarioKeys := make(map[string]string) for i := headerLineIdx + 2; i < len(lines); i++ { line := strings.TrimSpace(lines[i]) if line == "" { continue } // Skip separator rows (only dashes, pipes, colons - no backticks). if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") { continue } // Stop when we encounter the next section header. if strings.HasPrefix(line, "##") { break } cells := splitTableRow(line) scenarioName := strings.Trim(cells[colIndex["scenario"]], "`") keysCell := strings.TrimSpace(cells[colIndex["expectedOutputKeys"]]) scenarioKeys[scenarioName] = keysCell // Verify every required scenario row is present. found := false for _, req := range requiredScenarios { if req == scenarioName { found = true break } } if !found { // Not a required scenario; skip. continue } if keysCell == "" { t.Errorf("matrix row for scenario %q has an empty Expected output keys cell", scenarioName) } } // Ensure every required scenario is present in the matrix. for _, req := range requiredScenarios { if _, ok := scenarioKeys[req]; !ok { t.Errorf("required scenario %q is missing from the handoff matrix", req) } } // For each required scenario, parse keys from the matrix and verify fixture. for _, req := range requiredScenarios { keysCell, ok := scenarioKeys[req] if !ok { continue // reported above } promisedKeys := extractBacktickKeys(keysCell) if len(promisedKeys) == 0 { t.Errorf("scenario %q: matrix promises zero keys (empty Expected output keys cell)", req) continue } fixturePath := expectedFixturePath(req) f, err := os.Open(fixturePath) if err != nil { t.Fatalf("open expected fixture for %q: %v", req, err) } found := make(map[string]bool) scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Errorf("%s not valid JSON: %v", fixturePath, err) continue } for k := range rec { if !found[k] { for _, wanted := range promisedKeys { if k == wanted { found[k] = true } } } } } if err := scanner.Err(); err != nil { t.Fatalf("scan %q: %v", fixturePath, err) } f.Close() for _, pw := range promisedKeys { if !found[pw] { t.Errorf("%s (%q) missing promised key %q from matrix", fixturePath, req, pw) } } } } // extractBacktickKeys returns the set of backtick-wrapped strings from a // pipe-delimited markdown cell (e.g. "`a`, `b`, `c`"). func extractBacktickKeys(cell string) []string { var keys []string seen := make(map[string]bool) for _, part := range strings.Split(cell, ",") { trimmed := strings.TrimSpace(part) if strings.HasPrefix(trimmed, "`") && strings.HasSuffix(trimmed, "`") { key := trimmed[1 : len(trimmed)-1] if !seen[key] { seen[key] = true keys = append(keys, key) } } } return keys } // splitTableRow splits a markdown table row into its cell strings, trimming // the leading/trailing pipes and splitting on "|". func splitTableRow(line string) []string { line = strings.TrimPrefix(line, "|") line = strings.TrimSuffix(line, "|") var cells []string for _, part := range strings.Split(line, "|") { cells = append(cells, strings.TrimSpace(part)) } return cells } // matrixRow holds the parsed content of a single handoff matrix row. type matrixRow struct { Scenario string Command string InputFixture string ExpectedFixture string ExpectedOutputKeys string ExpectedExitCode string RepeatableOperation string UICandidate string UIDeferReason string CheckedField string } // mapCol accepts a list of column cells and a desired-name → index map, // and returns an int map from alias key to column index. func mapCol(cells []string, aliases map[string]string) map[string]int { out := make(map[string]int) for alias, name := range aliases { for i, cell := range cells { if cell == name { out[alias] = i break } } } return out } // TestHandoffDocumentedScenarioFixturesExist guards the handoff matrix against // referencing fixture paths that do not exist. Each required scenario must have // an input YAML fixture and an expected output fixture on disk, and the matrix // must reference both relative paths so the documented commands stay runnable. func TestHandoffDocumentedScenarioFixturesExist(t *testing.T) { matrix, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } doc := string(matrix) for _, s := range requiredScenarios { inputPath := filepath.Join("..", "..", "testdata", "operator", s+".yaml") if _, err := os.Stat(inputPath); err != nil { t.Errorf("input fixture for %q missing: %v", s, err) } if _, err := os.Stat(expectedFixturePath(s)); err != nil { t.Errorf("expected fixture for %q missing: %v", s, err) } inputRel := "testdata/operator/" + s + ".yaml" expectedRel := "testdata/operator/expected/" + s + ".jsonl" if !strings.Contains(doc, inputRel) { t.Errorf("handoff matrix does not reference input path %q", inputRel) } if !strings.Contains(doc, expectedRel) { t.Errorf("handoff matrix does not reference expected path %q", expectedRel) } } } // TestHandoffPaperRowsHaveRowLevelChecks validates that each paper-related // scenario (paper_trading_state and paper_order_lifecycle) has the following // row-level properties populated in the handoff matrix: // - Command is non-empty // - Input fixture is non-empty // - Expected fixture is non-empty // - Expected output keys is non-empty // - Repeatable operation is non-empty // - UI candidate is non-empty // - UI defer reason is non-empty // // The test also verifies that the Command cell references the Input fixture path. func TestHandoffPaperRowsHaveRowLevelChecks(t *testing.T) { data, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } lines := strings.Split(string(data), "\n") // Find header row headerIdx := -1 for i, line := range lines { if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") { headerIdx = i break } } if headerIdx < 0 { t.Fatal("matrix header row not found") } // Build column index map from header headerCells := splitTableRow(lines[headerIdx]) colIndex := mapCol(headerCells, map[string]string{ "scenario": "Scenario", "command": "Command", "inputFixture": "Input fixture", "expectedFixture": "Expected output fixture", "expectedOutputKeys": "Expected output keys", "repeatable": "Repeatable operation", "uiCandidate": "UI candidate", "uiDeferReason": "UI defer reason", }) // Validate all required columns exist requireCols := []string{"scenario", "command", "inputFixture", "expectedFixture", "expectedOutputKeys", "repeatable", "uiCandidate", "uiDeferReason"} for _, alias := range requireCols { if idx, ok := colIndex[alias]; !ok || idx < 0 { t.Errorf("required column alias %q not found in matrix header", alias) } } // Paper scenarios to validate paperScenarios := []string{"paper_trading_state", "paper_order_lifecycle"} // Track which paper scenarios are present in matrix present := make(map[string]bool) // Parse data rows for i := headerIdx + 2; i < len(lines); i++ { line := strings.TrimSpace(lines[i]) if line == "" { continue } // Skip separator rows if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") { continue } // Stop at next section header if strings.HasPrefix(line, "##") { break } cells := splitTableRow(line) scn := strings.Trim(cells[colIndex["scenario"]], "`") isPaper := false for _, ps := range paperScenarios { if scn == ps { isPaper = true present[ps] = true break } } if !isPaper { continue } // Check each required field is non-empty fields := []struct { alias string label string }{ {"command", "Command"}, {"inputFixture", "Input fixture"}, {"expectedFixture", "Expected output fixture"}, {"expectedOutputKeys", "Expected output keys"}, {"repeatable", "Repeatable operation"}, {"uiCandidate", "UI candidate"}, {"uiDeferReason", "UI defer reason"}, } for _, f := range fields { val := strings.TrimSpace(cells[colIndex[f.alias]]) if val == "" { t.Errorf("matrix row for scenario %q has empty %s cell", scn, f.label) } } // Verify Command references its Input fixture cmdCell := strings.TrimSpace(cells[colIndex["command"]]) inputCell := strings.TrimSpace(cells[colIndex["inputFixture"]]) // Strip backticks from both cells for the contains check cmdUnquoted := strings.Trim(cmdCell, "`") inputUnquoted := strings.Trim(inputCell, "`") if !strings.Contains(cmdUnquoted, inputUnquoted) { t.Errorf("scenario %q: Command %q does not reference Input fixture %q", scn, cmdUnquoted, inputUnquoted) } } // Ensure all paper scenarios are present for _, ps := range paperScenarios { if !present[ps] { t.Errorf("paper scenario %q missing from handoff matrix", ps) } } } func parseExpectedExitCodes(cell string) []int { var codes []int parts := strings.Split(cell, "`") for i := 1; i < len(parts); i += 2 { val := strings.TrimSpace(parts[i]) var code int if _, err := fmt.Sscanf(val, "%d", &code); err == nil { codes = append(codes, code) } } return codes } func TestHandoffExpectedFixturesExitCodeAndStatus(t *testing.T) { data, err := os.ReadFile(handoffMatrixPath()) if err != nil { t.Fatalf("read handoff matrix: %v", err) } lines := strings.Split(string(data), "\n") headerLineIdx := -1 for i, line := range lines { if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") { headerLineIdx = i break } } if headerLineIdx < 0 { t.Fatal("matrix header row not found in handoff doc") } headerCells := splitTableRow(lines[headerLineIdx]) colIndex := mapCol(headerCells, map[string]string{ "scenario": "Scenario", "expectedExitCode": "Expected exit code", }) for i := headerLineIdx + 2; i < len(lines); i++ { line := strings.TrimSpace(lines[i]) if line == "" { continue } if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") { continue } if strings.HasPrefix(line, "##") { break } cells := splitTableRow(line) scenarioName := strings.Trim(cells[colIndex["scenario"]], "`") isRequired := false for _, req := range requiredScenarios { if req == scenarioName { isRequired = true break } } if !isRequired { continue } exitCodeCell := cells[colIndex["expectedExitCode"]] expectedCodes := parseExpectedExitCodes(exitCodeCell) if len(expectedCodes) == 0 { t.Errorf("scenario %q: matrix specifies zero expected exit codes in cell %q", scenarioName, exitCodeCell) continue } fixturePath := expectedFixturePath(scenarioName) f, err := os.Open(fixturePath) if err != nil { t.Fatalf("open expected fixture for %q: %v", scenarioName, err) } var summaryRec map[string]any scanner := bufio.NewScanner(f) for scanner.Scan() { text := strings.TrimSpace(scanner.Text()) if text == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(text), &rec); err != nil { t.Errorf("%s not valid JSON: %v", fixturePath, err) continue } if rec["type"] == "summary" { summaryRec = rec } } f.Close() if summaryRec == nil { t.Errorf("%s has no summary line", fixturePath) continue } exitCodeVal, ok := summaryRec["exit_code"] if !ok { t.Errorf("%s summary line missing exit_code field", fixturePath) continue } exitCodeFloat, ok := exitCodeVal.(float64) if !ok { t.Errorf("%s summary line exit_code is not a number: %T (%v)", fixturePath, exitCodeVal, exitCodeVal) continue } actualExitCode := int(exitCodeFloat) matched := false for _, want := range expectedCodes { if actualExitCode == want { matched = true break } } if !matched { t.Errorf("%s summary exit_code = %d, want one of %v from matrix", fixturePath, actualExitCode, expectedCodes) } statusVal, ok := summaryRec["status"] if !ok { t.Errorf("%s summary line missing status field", fixturePath) continue } actualStatus, _ := statusVal.(string) switch scenarioName { case "backtest_matrix_success", "backtest_matrix_failed": if actualExitCode != 0 { t.Errorf("%s: matrix success/failed row must have exit_code = 0 (got %d)", scenarioName, actualExitCode) } if actualStatus != "ok" { t.Errorf("%s: matrix success/failed row must have status = \"ok\" (got %q)", scenarioName, actualStatus) } case "backtest_matrix_timeout": if actualExitCode != 3 { t.Errorf("%s: matrix timeout row must have exit_code = 3 (got %d)", scenarioName, actualExitCode) } if actualStatus != "transport_error" { t.Errorf("%s: matrix timeout row must have status = \"transport_error\" (got %q)", scenarioName, actualStatus) } case "backtest_matrix_mismatch": if actualExitCode != 1 { t.Errorf("%s: matrix mismatch row must have exit_code = 1 (got %d)", scenarioName, actualExitCode) } if actualStatus != "failed" { t.Errorf("%s: matrix mismatch row must have status = \"failed\" (got %q)", scenarioName, actualStatus) } } } }