864 lines
27 KiB
Go
864 lines
27 KiB
Go
package operator
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// requiredScenarios are the six headless validation scenarios the milestone
|
|
// handoff table requires evidence for. The handoff matrix and the expected
|
|
// output fixtures must cover all of them so the Flutter Operator Console MVP
|
|
// can validate each workflow before building any screen.
|
|
var requiredScenarios = []string{
|
|
"api_connection_smoke",
|
|
"market_data_status_query",
|
|
"us_market_data_status_query",
|
|
"backtest_run_request",
|
|
"backtest_run_polling",
|
|
"backtest_result_summary",
|
|
"backtest_matrix_success",
|
|
"backtest_matrix_failed",
|
|
"backtest_matrix_timeout",
|
|
"backtest_matrix_mismatch",
|
|
"invalid_request_matrix",
|
|
"kis_daily_import_smoke",
|
|
"minute_import_rejected",
|
|
"paper_trading_state",
|
|
"paper_order_lifecycle",
|
|
"live_order_lifecycle",
|
|
"scheduler_refresh_status",
|
|
"scheduled_backtest_readiness",
|
|
}
|
|
|
|
func handoffMatrixPath() string {
|
|
return filepath.Join("..", "..", "testdata", "operator", "headless_validation.md")
|
|
}
|
|
|
|
func expectedFixturePath(scenario string) string {
|
|
return filepath.Join("..", "..", "testdata", "operator", "expected", scenario+".jsonl")
|
|
}
|
|
|
|
// TestHeadlessValidationHandoffCoversRequiredScenarios fails if the handoff
|
|
// matrix drops any required scenario, which would leave the Flutter MVP without
|
|
// command/field evidence for that workflow.
|
|
func TestHeadlessValidationHandoffCoversRequiredScenarios(t *testing.T) {
|
|
data, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
doc := string(data)
|
|
for _, s := range requiredScenarios {
|
|
if !strings.Contains(doc, "`"+s+"`") {
|
|
t.Errorf("handoff matrix missing scenario %q", s)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestExpectedOutputFixturesAreValidJSONLines checks that every expected output
|
|
// fixture parses as JSON lines and that each record carries the machine-readable
|
|
// keys the handoff contract promises: scenario/status on every line, action on
|
|
// step lines, plus at least one step and one summary line.
|
|
func TestExpectedOutputFixturesAreValidJSONLines(t *testing.T) {
|
|
for _, s := range requiredScenarios {
|
|
path := expectedFixturePath(s)
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
t.Errorf("open expected fixture %q: %v", path, err)
|
|
continue
|
|
}
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
lineNo := 0
|
|
sawStep := false
|
|
sawSummary := false
|
|
for scanner.Scan() {
|
|
lineNo++
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Errorf("%s:%d not valid JSON: %v", path, lineNo, err)
|
|
continue
|
|
}
|
|
|
|
if rec["scenario"] != s {
|
|
t.Errorf("%s:%d scenario = %v, want %q", path, lineNo, rec["scenario"], s)
|
|
}
|
|
if _, ok := rec["status"]; !ok {
|
|
t.Errorf("%s:%d missing status field", path, lineNo)
|
|
}
|
|
|
|
switch rec["type"] {
|
|
case "step":
|
|
sawStep = true
|
|
if _, ok := rec["action"]; !ok {
|
|
t.Errorf("%s:%d step line missing action field", path, lineNo)
|
|
}
|
|
case "summary":
|
|
sawSummary = true
|
|
default:
|
|
t.Errorf("%s:%d unexpected type %v", path, lineNo, rec["type"])
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Errorf("scan %q: %v", path, err)
|
|
}
|
|
f.Close()
|
|
|
|
if !sawStep {
|
|
t.Errorf("%s has no step line", path)
|
|
}
|
|
if !sawSummary {
|
|
t.Errorf("%s has no summary line", path)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestStatusScenarioFixtureImportsBeforeReads guards the command-first status
|
|
// scenario: its expected fixture must carry an import_daily_bars step line with
|
|
// instrument_count and bar_count keys before the instrument/bar read lines, so
|
|
// the handoff evidence shows imported counts rather than a bare status read.
|
|
func TestStatusScenarioFixtureImportsBeforeReads(t *testing.T) {
|
|
const scenario = "market_data_status_query"
|
|
f, err := os.Open(expectedFixturePath(scenario))
|
|
if err != nil {
|
|
t.Fatalf("open expected fixture: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
importLine := -1
|
|
instLine := -1
|
|
barLine := -1
|
|
lineNo := 0
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
lineNo++
|
|
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatalf("line %d not valid JSON: %v", lineNo, err)
|
|
}
|
|
switch rec["action"] {
|
|
case "import_daily_bars":
|
|
importLine = lineNo
|
|
if _, ok := rec["instrument_count"]; !ok {
|
|
t.Errorf("import step line %d missing instrument_count", lineNo)
|
|
}
|
|
if _, ok := rec["bar_count"]; !ok {
|
|
t.Errorf("import step line %d missing bar_count", lineNo)
|
|
}
|
|
case "list_instruments":
|
|
instLine = lineNo
|
|
case "list_bars":
|
|
barLine = lineNo
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Fatalf("scan fixture: %v", err)
|
|
}
|
|
|
|
if importLine < 0 {
|
|
t.Fatal("status fixture has no import_daily_bars step line")
|
|
}
|
|
if instLine < 0 || barLine < 0 {
|
|
t.Fatal("status fixture missing instrument/bar read step lines")
|
|
}
|
|
if importLine > instLine || importLine > barLine {
|
|
t.Errorf("import line %d must precede read lines (instruments=%d, bars=%d)", importLine, instLine, barLine)
|
|
}
|
|
}
|
|
|
|
func TestUSStatusScenarioFixtureImportsBeforeReads(t *testing.T) {
|
|
const scenario = "us_market_data_status_query"
|
|
f, err := os.Open(expectedFixturePath(scenario))
|
|
if err != nil {
|
|
t.Fatalf("open expected fixture: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
importLine := -1
|
|
instLine := -1
|
|
barLine := -1
|
|
lineNo := 0
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
lineNo++
|
|
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatalf("line %d not valid JSON: %v", lineNo, err)
|
|
}
|
|
switch rec["action"] {
|
|
case "import_daily_bars":
|
|
importLine = lineNo
|
|
if _, ok := rec["instrument_count"]; !ok {
|
|
t.Errorf("import step line %d missing instrument_count", lineNo)
|
|
}
|
|
if _, ok := rec["bar_count"]; !ok {
|
|
t.Errorf("import step line %d missing bar_count", lineNo)
|
|
}
|
|
case "list_instruments":
|
|
instLine = lineNo
|
|
case "list_bars":
|
|
barLine = lineNo
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Fatalf("scan fixture: %v", err)
|
|
}
|
|
|
|
if importLine < 0 {
|
|
t.Fatal("status fixture has no import_daily_bars step line")
|
|
}
|
|
if instLine < 0 || barLine < 0 {
|
|
t.Fatal("status fixture missing instrument/bar read step lines")
|
|
}
|
|
if importLine > instLine || importLine > barLine {
|
|
t.Errorf("import line %d must precede read lines (instruments=%d, bars=%d)", importLine, instLine, barLine)
|
|
}
|
|
}
|
|
|
|
// requiredScenarioKeys are the expected output keys each required scenario's
|
|
// JSONL fixture must carry. Kept as a fallback so existing tests still pass,
|
|
// but the definitive source of truth is now the handoff matrix itself via
|
|
// TestHandoffExpectedFixturesWithMatrixPromisedKeys.
|
|
var requiredScenarioKeys = map[string][]string{
|
|
"api_connection_smoke": {"scenario", "status", "type", "action"},
|
|
"backtest_result_summary": {"run_id", "run_status", "starting_cash", "ending_equity", "total_return", "trade_count"},
|
|
"backtest_run_polling": {"run_id", "run_status", "provider", "bar_count", "instrument_count"},
|
|
"backtest_run_request": {"run_id", "run_status", "bar_count", "instrument_count", "provider"},
|
|
"backtest_matrix_success": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"},
|
|
"backtest_matrix_failed": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"},
|
|
"backtest_matrix_timeout": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"},
|
|
"backtest_matrix_mismatch": {"matrix_run_id", "universe", "strategy_id", "timeframe", "period_id", "run_id", "run_status", "exit_code"},
|
|
"invalid_request_matrix": {"scenario", "status", "type", "action", "error_code", "error_message"},
|
|
"kis_daily_import_smoke": {"provider", "instrument_count", "bar_count"},
|
|
"minute_import_rejected": {"scenario", "status", "type", "action", "error_code", "error_message"},
|
|
"market_data_status_query": {"provider", "instrument_count", "bar_count", "count", "universe", "freshness_status", "missing_count", "latest_yyyymmdd", "gap_status", "gap_count", "duplicate_count", "provider_delay_days"},
|
|
"us_market_data_status_query": {"provider", "instrument_count", "bar_count", "count", "universe", "freshness_status", "missing_count", "latest_yyyymmdd", "gap_status", "gap_count", "duplicate_count", "provider_delay_days"},
|
|
"paper_trading_state": {"account_id", "run_id", "run_status", "cash", "equity_point_count", "fill_count", "latest_equity", "position_count", "risk"},
|
|
"paper_order_lifecycle": {"scenario", "status", "type", "action", "account_id", "run_id", "run_status", "cash", "equity_point_count", "fill_count", "latest_equity", "position_count", "risk", "order_id", "order_status", "fill_price"},
|
|
"live_order_lifecycle": {"scenario", "status", "type", "action", "live_order_id", "live_order_status"},
|
|
"scheduler_refresh_status": {"scenario", "status", "type", "action", "schedule", "scheduler_status", "last_success_unix_ms", "last_error", "next_run_unix_ms", "imported_bar_count", "missing_count", "gap_count", "duplicate_count", "provider_delay_days"},
|
|
"scheduled_backtest_readiness": {"scenario", "status", "type", "action", "scheduler_status", "imported_bar_count", "last_success_unix_ms", "next_run_unix_ms", "run_id", "run_status", "starting_cash", "ending_equity", "total_return", "trade_count"},
|
|
}
|
|
|
|
// TestHandoffMatrixDocumentsCommandFirstColumns asserts that the handoff
|
|
// matrix has been updated from the legacy wireframe-only columns to the
|
|
// command-first columns introduced in the current milestone. The matrix must
|
|
// contain explicit section text and column headers for expected output keys,
|
|
// repeatable operation, UI candidate, and UI defer reason.
|
|
func TestHandoffMatrixDocumentsCommandFirstColumns(t *testing.T) {
|
|
data, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
doc := string(data)
|
|
|
|
requiredHeaders := []string{
|
|
"Expected output keys",
|
|
"Repeatable operation",
|
|
"UI candidate",
|
|
"UI defer reason",
|
|
}
|
|
for _, header := range requiredHeaders {
|
|
if !strings.Contains(doc, header) {
|
|
t.Errorf("handoff matrix missing required column header %q", header)
|
|
}
|
|
}
|
|
|
|
// Also verify the new section is present (not just the old header text)
|
|
if !strings.Contains(doc, "command-first workflow handoff matrix") {
|
|
t.Error("handoff matrix missing 'command-first workflow handoff matrix' section")
|
|
}
|
|
}
|
|
|
|
// TestHandoffExpectedFixturesContainPromisedKeys asserts that every required scenario's
|
|
// JSONL fixture contains at least one line with each key promised in the
|
|
// handoff matrix. Kept as a secondary guard using requiredScenarioKeys.
|
|
func TestHandoffExpectedFixturesContainPromisedKeys(t *testing.T) {
|
|
// Ensure the requiredScenarios slice matches our expectedScenarioKeys map
|
|
if len(requiredScenarios) == 0 {
|
|
t.Fatal("requiredScenarios must not be empty")
|
|
}
|
|
|
|
for _, s := range requiredScenarios {
|
|
keys, ok := requiredScenarioKeys[s]
|
|
if !ok {
|
|
t.Fatalf("required scenario %q has no promised keys defined in requiredScenarioKeys", s)
|
|
}
|
|
if len(keys) == 0 {
|
|
t.Fatalf("required scenario %q has an empty promised keys list in requiredScenarioKeys", s)
|
|
}
|
|
|
|
path := expectedFixturePath(s)
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
t.Fatalf("open expected fixture for %q: %v", s, err)
|
|
}
|
|
|
|
found := make(map[string]bool)
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Errorf("%s not valid JSON: %v", path, err)
|
|
continue
|
|
}
|
|
for k := range rec {
|
|
if found[k] {
|
|
continue
|
|
}
|
|
for _, want := range keys {
|
|
if k == want {
|
|
found[k] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Fatalf("scan %q: %v", path, err)
|
|
}
|
|
f.Close()
|
|
|
|
for _, want := range keys {
|
|
if !found[want] {
|
|
t.Errorf("%s (%q) missing promised key %q", path, s, want)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestHandoffExpectedFixturesWithMatrixPromisedKeys reads the command-first handoff
|
|
// matrix from the handoff doc, finds the "Expected output keys" cell for each
|
|
// required scenario row, extracts backtick-wrapped key names, and verifies
|
|
// each promised key appears at least once in that scenario's expected JSONL
|
|
// fixture. The test fails loudly when:
|
|
// - a required scenario row is missing from the matrix,
|
|
// - the matrix promises zero keys (empty Expected output keys cell),
|
|
// - a promised key is absent from the expected fixture.
|
|
//
|
|
// This is the definitive matrix-driven key contract test introduced by
|
|
// local-G06 so the test source of truth is the handoff doc, not a hardcoded
|
|
// map.
|
|
func TestHandoffExpectedFixturesWithMatrixPromisedKeys(t *testing.T) {
|
|
data, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
lines := strings.Split(string(data), "\n")
|
|
|
|
// Find the table header: first line containing both "Scenario" and "Command"
|
|
headerLineIdx := -1
|
|
for i, line := range lines {
|
|
if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") {
|
|
headerLineIdx = i
|
|
break
|
|
}
|
|
}
|
|
if headerLineIdx < 0 {
|
|
t.Fatal("matrix header row not found in handoff doc")
|
|
}
|
|
|
|
// The next line is the separator row (|---|---|...), skip it.
|
|
// Data rows start at headerLineIdx+2.
|
|
|
|
// Build a column-index map from the header row.
|
|
headerCells := splitTableRow(lines[headerLineIdx])
|
|
colIndex := mapCol(headerCells, map[string]string{
|
|
"scenario": "Scenario",
|
|
"expectedOutputKeys": "Expected output keys",
|
|
})
|
|
|
|
// Iterate the matrix data rows (start after the separator row after header).
|
|
scenarioKeys := make(map[string]string)
|
|
for i := headerLineIdx + 2; i < len(lines); i++ {
|
|
line := strings.TrimSpace(lines[i])
|
|
if line == "" {
|
|
continue
|
|
}
|
|
// Skip separator rows (only dashes, pipes, colons - no backticks).
|
|
if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") {
|
|
continue
|
|
}
|
|
// Stop when we encounter the next section header.
|
|
if strings.HasPrefix(line, "##") {
|
|
break
|
|
}
|
|
cells := splitTableRow(line)
|
|
scenarioName := strings.Trim(cells[colIndex["scenario"]], "`")
|
|
keysCell := strings.TrimSpace(cells[colIndex["expectedOutputKeys"]])
|
|
scenarioKeys[scenarioName] = keysCell
|
|
|
|
// Verify every required scenario row is present.
|
|
found := false
|
|
for _, req := range requiredScenarios {
|
|
if req == scenarioName {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
// Not a required scenario; skip.
|
|
continue
|
|
}
|
|
|
|
if keysCell == "" {
|
|
t.Errorf("matrix row for scenario %q has an empty Expected output keys cell", scenarioName)
|
|
}
|
|
}
|
|
|
|
// Ensure every required scenario is present in the matrix.
|
|
for _, req := range requiredScenarios {
|
|
if _, ok := scenarioKeys[req]; !ok {
|
|
t.Errorf("required scenario %q is missing from the handoff matrix", req)
|
|
}
|
|
}
|
|
|
|
// For each required scenario, parse keys from the matrix and verify fixture.
|
|
for _, req := range requiredScenarios {
|
|
keysCell, ok := scenarioKeys[req]
|
|
if !ok {
|
|
continue // reported above
|
|
}
|
|
|
|
promisedKeys := extractBacktickKeys(keysCell)
|
|
if len(promisedKeys) == 0 {
|
|
t.Errorf("scenario %q: matrix promises zero keys (empty Expected output keys cell)", req)
|
|
continue
|
|
}
|
|
|
|
fixturePath := expectedFixturePath(req)
|
|
f, err := os.Open(fixturePath)
|
|
if err != nil {
|
|
t.Fatalf("open expected fixture for %q: %v", req, err)
|
|
}
|
|
|
|
found := make(map[string]bool)
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Errorf("%s not valid JSON: %v", fixturePath, err)
|
|
continue
|
|
}
|
|
for k := range rec {
|
|
if !found[k] {
|
|
for _, wanted := range promisedKeys {
|
|
if k == wanted {
|
|
found[k] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Fatalf("scan %q: %v", fixturePath, err)
|
|
}
|
|
f.Close()
|
|
|
|
for _, pw := range promisedKeys {
|
|
if !found[pw] {
|
|
t.Errorf("%s (%q) missing promised key %q from matrix", fixturePath, req, pw)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractBacktickKeys returns the set of backtick-wrapped strings from a
|
|
// pipe-delimited markdown cell (e.g. "`a`, `b`, `c`").
|
|
func extractBacktickKeys(cell string) []string {
|
|
var keys []string
|
|
seen := make(map[string]bool)
|
|
for _, part := range strings.Split(cell, ",") {
|
|
trimmed := strings.TrimSpace(part)
|
|
if strings.HasPrefix(trimmed, "`") && strings.HasSuffix(trimmed, "`") {
|
|
key := trimmed[1 : len(trimmed)-1]
|
|
if !seen[key] {
|
|
seen[key] = true
|
|
keys = append(keys, key)
|
|
}
|
|
}
|
|
}
|
|
return keys
|
|
}
|
|
|
|
// splitTableRow splits a markdown table row into its cell strings, trimming
|
|
// the leading/trailing pipes and splitting on "|".
|
|
func splitTableRow(line string) []string {
|
|
line = strings.TrimPrefix(line, "|")
|
|
line = strings.TrimSuffix(line, "|")
|
|
var cells []string
|
|
for _, part := range strings.Split(line, "|") {
|
|
cells = append(cells, strings.TrimSpace(part))
|
|
}
|
|
return cells
|
|
}
|
|
|
|
// matrixRow holds the parsed content of a single handoff matrix row.
|
|
type matrixRow struct {
|
|
Scenario string
|
|
Command string
|
|
InputFixture string
|
|
ExpectedFixture string
|
|
ExpectedOutputKeys string
|
|
ExpectedExitCode string
|
|
RepeatableOperation string
|
|
UICandidate string
|
|
UIDeferReason string
|
|
CheckedField string
|
|
}
|
|
|
|
// mapCol accepts a list of column cells and a desired-name → index map,
|
|
// and returns an int map from alias key to column index.
|
|
func mapCol(cells []string, aliases map[string]string) map[string]int {
|
|
out := make(map[string]int)
|
|
for alias, name := range aliases {
|
|
for i, cell := range cells {
|
|
if cell == name {
|
|
out[alias] = i
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestHandoffDocumentedScenarioFixturesExist guards the handoff matrix against
|
|
// referencing fixture paths that do not exist. Each required scenario must have
|
|
// an input YAML fixture and an expected output fixture on disk, and the matrix
|
|
// must reference both relative paths so the documented commands stay runnable.
|
|
func TestHandoffDocumentedScenarioFixturesExist(t *testing.T) {
|
|
matrix, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
doc := string(matrix)
|
|
|
|
for _, s := range requiredScenarios {
|
|
inputPath := filepath.Join("..", "..", "testdata", "operator", s+".yaml")
|
|
if _, err := os.Stat(inputPath); err != nil {
|
|
t.Errorf("input fixture for %q missing: %v", s, err)
|
|
}
|
|
if _, err := os.Stat(expectedFixturePath(s)); err != nil {
|
|
t.Errorf("expected fixture for %q missing: %v", s, err)
|
|
}
|
|
|
|
inputRel := "testdata/operator/" + s + ".yaml"
|
|
expectedRel := "testdata/operator/expected/" + s + ".jsonl"
|
|
if !strings.Contains(doc, inputRel) {
|
|
t.Errorf("handoff matrix does not reference input path %q", inputRel)
|
|
}
|
|
if !strings.Contains(doc, expectedRel) {
|
|
t.Errorf("handoff matrix does not reference expected path %q", expectedRel)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestHandoffPaperRowsHaveRowLevelChecks validates that each paper-related
|
|
// scenario (paper_trading_state and paper_order_lifecycle) has the following
|
|
// row-level properties populated in the handoff matrix:
|
|
// - Command is non-empty
|
|
// - Input fixture is non-empty
|
|
// - Expected fixture is non-empty
|
|
// - Expected output keys is non-empty
|
|
// - Repeatable operation is non-empty
|
|
// - UI candidate is non-empty
|
|
// - UI defer reason is non-empty
|
|
//
|
|
// The test also verifies that the Command cell references the Input fixture path.
|
|
func TestHandoffPaperRowsHaveRowLevelChecks(t *testing.T) {
|
|
data, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
lines := strings.Split(string(data), "\n")
|
|
|
|
// Find header row
|
|
headerIdx := -1
|
|
for i, line := range lines {
|
|
if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") {
|
|
headerIdx = i
|
|
break
|
|
}
|
|
}
|
|
if headerIdx < 0 {
|
|
t.Fatal("matrix header row not found")
|
|
}
|
|
|
|
// Build column index map from header
|
|
headerCells := splitTableRow(lines[headerIdx])
|
|
colIndex := mapCol(headerCells, map[string]string{
|
|
"scenario": "Scenario",
|
|
"command": "Command",
|
|
"inputFixture": "Input fixture",
|
|
"expectedFixture": "Expected output fixture",
|
|
"expectedOutputKeys": "Expected output keys",
|
|
"repeatable": "Repeatable operation",
|
|
"uiCandidate": "UI candidate",
|
|
"uiDeferReason": "UI defer reason",
|
|
})
|
|
|
|
// Validate all required columns exist
|
|
requireCols := []string{"scenario", "command", "inputFixture", "expectedFixture",
|
|
"expectedOutputKeys", "repeatable", "uiCandidate", "uiDeferReason"}
|
|
for _, alias := range requireCols {
|
|
if idx, ok := colIndex[alias]; !ok || idx < 0 {
|
|
t.Errorf("required column alias %q not found in matrix header", alias)
|
|
}
|
|
}
|
|
|
|
// Paper scenarios to validate
|
|
paperScenarios := []string{"paper_trading_state", "paper_order_lifecycle"}
|
|
|
|
// Track which paper scenarios are present in matrix
|
|
present := make(map[string]bool)
|
|
|
|
// Parse data rows
|
|
for i := headerIdx + 2; i < len(lines); i++ {
|
|
line := strings.TrimSpace(lines[i])
|
|
if line == "" {
|
|
continue
|
|
}
|
|
// Skip separator rows
|
|
if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") {
|
|
continue
|
|
}
|
|
// Stop at next section header
|
|
if strings.HasPrefix(line, "##") {
|
|
break
|
|
}
|
|
|
|
cells := splitTableRow(line)
|
|
scn := strings.Trim(cells[colIndex["scenario"]], "`")
|
|
|
|
isPaper := false
|
|
for _, ps := range paperScenarios {
|
|
if scn == ps {
|
|
isPaper = true
|
|
present[ps] = true
|
|
break
|
|
}
|
|
}
|
|
if !isPaper {
|
|
continue
|
|
}
|
|
|
|
// Check each required field is non-empty
|
|
fields := []struct {
|
|
alias string
|
|
label string
|
|
}{
|
|
{"command", "Command"},
|
|
{"inputFixture", "Input fixture"},
|
|
{"expectedFixture", "Expected output fixture"},
|
|
{"expectedOutputKeys", "Expected output keys"},
|
|
{"repeatable", "Repeatable operation"},
|
|
{"uiCandidate", "UI candidate"},
|
|
{"uiDeferReason", "UI defer reason"},
|
|
}
|
|
for _, f := range fields {
|
|
val := strings.TrimSpace(cells[colIndex[f.alias]])
|
|
if val == "" {
|
|
t.Errorf("matrix row for scenario %q has empty %s cell", scn, f.label)
|
|
}
|
|
}
|
|
|
|
// Verify Command references its Input fixture
|
|
cmdCell := strings.TrimSpace(cells[colIndex["command"]])
|
|
inputCell := strings.TrimSpace(cells[colIndex["inputFixture"]])
|
|
// Strip backticks from both cells for the contains check
|
|
cmdUnquoted := strings.Trim(cmdCell, "`")
|
|
inputUnquoted := strings.Trim(inputCell, "`")
|
|
if !strings.Contains(cmdUnquoted, inputUnquoted) {
|
|
t.Errorf("scenario %q: Command %q does not reference Input fixture %q", scn, cmdUnquoted, inputUnquoted)
|
|
}
|
|
}
|
|
|
|
// Ensure all paper scenarios are present
|
|
for _, ps := range paperScenarios {
|
|
if !present[ps] {
|
|
t.Errorf("paper scenario %q missing from handoff matrix", ps)
|
|
}
|
|
}
|
|
}
|
|
|
|
func parseExpectedExitCodes(cell string) []int {
|
|
var codes []int
|
|
parts := strings.Split(cell, "`")
|
|
for i := 1; i < len(parts); i += 2 {
|
|
val := strings.TrimSpace(parts[i])
|
|
var code int
|
|
if _, err := fmt.Sscanf(val, "%d", &code); err == nil {
|
|
codes = append(codes, code)
|
|
}
|
|
}
|
|
return codes
|
|
}
|
|
|
|
func TestHandoffExpectedFixturesExitCodeAndStatus(t *testing.T) {
|
|
data, err := os.ReadFile(handoffMatrixPath())
|
|
if err != nil {
|
|
t.Fatalf("read handoff matrix: %v", err)
|
|
}
|
|
lines := strings.Split(string(data), "\n")
|
|
|
|
headerLineIdx := -1
|
|
for i, line := range lines {
|
|
if strings.Contains(line, "| Scenario") && strings.Contains(line, "Command") {
|
|
headerLineIdx = i
|
|
break
|
|
}
|
|
}
|
|
if headerLineIdx < 0 {
|
|
t.Fatal("matrix header row not found in handoff doc")
|
|
}
|
|
|
|
headerCells := splitTableRow(lines[headerLineIdx])
|
|
colIndex := mapCol(headerCells, map[string]string{
|
|
"scenario": "Scenario",
|
|
"expectedExitCode": "Expected exit code",
|
|
})
|
|
|
|
for i := headerLineIdx + 2; i < len(lines); i++ {
|
|
line := strings.TrimSpace(lines[i])
|
|
if line == "" {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(line, "|") && !strings.Contains(line, "`") && strings.Contains(line, "-") {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(line, "##") {
|
|
break
|
|
}
|
|
|
|
cells := splitTableRow(line)
|
|
scenarioName := strings.Trim(cells[colIndex["scenario"]], "`")
|
|
|
|
isRequired := false
|
|
for _, req := range requiredScenarios {
|
|
if req == scenarioName {
|
|
isRequired = true
|
|
break
|
|
}
|
|
}
|
|
if !isRequired {
|
|
continue
|
|
}
|
|
|
|
exitCodeCell := cells[colIndex["expectedExitCode"]]
|
|
expectedCodes := parseExpectedExitCodes(exitCodeCell)
|
|
if len(expectedCodes) == 0 {
|
|
t.Errorf("scenario %q: matrix specifies zero expected exit codes in cell %q", scenarioName, exitCodeCell)
|
|
continue
|
|
}
|
|
|
|
fixturePath := expectedFixturePath(scenarioName)
|
|
f, err := os.Open(fixturePath)
|
|
if err != nil {
|
|
t.Fatalf("open expected fixture for %q: %v", scenarioName, err)
|
|
}
|
|
|
|
var summaryRec map[string]any
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
text := strings.TrimSpace(scanner.Text())
|
|
if text == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(text), &rec); err != nil {
|
|
t.Errorf("%s not valid JSON: %v", fixturePath, err)
|
|
continue
|
|
}
|
|
if rec["type"] == "summary" {
|
|
summaryRec = rec
|
|
}
|
|
}
|
|
f.Close()
|
|
|
|
if summaryRec == nil {
|
|
t.Errorf("%s has no summary line", fixturePath)
|
|
continue
|
|
}
|
|
|
|
exitCodeVal, ok := summaryRec["exit_code"]
|
|
if !ok {
|
|
t.Errorf("%s summary line missing exit_code field", fixturePath)
|
|
continue
|
|
}
|
|
|
|
exitCodeFloat, ok := exitCodeVal.(float64)
|
|
if !ok {
|
|
t.Errorf("%s summary line exit_code is not a number: %T (%v)", fixturePath, exitCodeVal, exitCodeVal)
|
|
continue
|
|
}
|
|
actualExitCode := int(exitCodeFloat)
|
|
|
|
matched := false
|
|
for _, want := range expectedCodes {
|
|
if actualExitCode == want {
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
t.Errorf("%s summary exit_code = %d, want one of %v from matrix", fixturePath, actualExitCode, expectedCodes)
|
|
}
|
|
|
|
statusVal, ok := summaryRec["status"]
|
|
if !ok {
|
|
t.Errorf("%s summary line missing status field", fixturePath)
|
|
continue
|
|
}
|
|
actualStatus, _ := statusVal.(string)
|
|
|
|
switch scenarioName {
|
|
case "backtest_matrix_success", "backtest_matrix_failed":
|
|
if actualExitCode != 0 {
|
|
t.Errorf("%s: matrix success/failed row must have exit_code = 0 (got %d)", scenarioName, actualExitCode)
|
|
}
|
|
if actualStatus != "ok" {
|
|
t.Errorf("%s: matrix success/failed row must have status = \"ok\" (got %q)", scenarioName, actualStatus)
|
|
}
|
|
case "backtest_matrix_timeout":
|
|
if actualExitCode != 3 {
|
|
t.Errorf("%s: matrix timeout row must have exit_code = 3 (got %d)", scenarioName, actualExitCode)
|
|
}
|
|
if actualStatus != "transport_error" {
|
|
t.Errorf("%s: matrix timeout row must have status = \"transport_error\" (got %q)", scenarioName, actualStatus)
|
|
}
|
|
case "backtest_matrix_mismatch":
|
|
if actualExitCode != 1 {
|
|
t.Errorf("%s: matrix mismatch row must have exit_code = 1 (got %d)", scenarioName, actualExitCode)
|
|
}
|
|
if actualStatus != "failed" {
|
|
t.Errorf("%s: matrix mismatch row must have status = \"failed\" (got %q)", scenarioName, actualStatus)
|
|
}
|
|
}
|
|
}
|
|
}
|