package operator import ( "bytes" "context" "fmt" "strings" "sync" "testing" "time" altv1 "git.toki-labs.com/toki/alt/packages/contracts/gen/go/alt/v1" ) func TestRunBacktestRequestOutputsRunID(t *testing.T) { api := &fakeAPI{ startBacktestResp: &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{ Id: "run-12345", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING, }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_run_request", Steps: []Step{ { ID: "start", Action: ActionStartBacktest, Request: Request{ StrategyID: "strategy-v1", Market: "kr", Timeframe: "daily", FromUnixMs: 1746057600000, ToUnixMs: 1747267200000, Symbols: []string{"005930"}, }, Expect: Expect{ Status: "ok", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } if !strings.Contains(out, "run.id=run-12345") { t.Errorf("output %q missing run.id=run-12345", out) } if !strings.Contains(out, "run.status=pending") { t.Errorf("output %q missing run.status=pending", out) } got := api.lastStartBacktestReq() if got == nil { t.Fatal("server did not receive a start backtest request") } if got.GetSpec().GetStrategyId() != "strategy-v1" { t.Errorf("server received strategy_id = %q, want strategy-v1", got.GetSpec().GetStrategyId()) } if len(got.GetSpec().GetSelector().GetSymbols()) != 1 || got.GetSpec().GetSelector().GetSymbols()[0] != "005930" { t.Errorf("server received symbols = %+v, want [005930]", got.GetSpec().GetSelector().GetSymbols()) } } func TestRunBacktestImportThenStartFlow(t *testing.T) { api := &fakeAPI{ importDailyBarsResp: &altv1.ImportDailyBarsResponse{Provider: "kis", InstrumentCount: 1, BarCount: 42}, startBacktestResp: &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{ Id: "run-12345", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING, }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_run_request", Steps: []Step{ { ID: "import_kr_daily_bars", Action: ActionImportDailyBars, Request: Request{ Provider: "kis", SelectorKind: "watchlist", Market: "kr", Venue: "krx", Symbols: []string{"005930"}, FromYYYYMMDD: "20240527", ToYYYYMMDD: "20240528", }, Expect: Expect{Status: "ok"}, }, { ID: "start_run", Action: ActionStartBacktest, Request: Request{ StrategyID: "strategy-v1", Market: "kr", Timeframe: "daily", FromUnixMs: 1746057600000, ToUnixMs: 1747267200000, Symbols: []string{"005930"}, }, Expect: Expect{Status: "ok"}, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } // The import step runs before the start step and reports imported counts. if !strings.Contains(out, "instrument_count=1") || !strings.Contains(out, "bar_count=42") { t.Errorf("output %q missing import counts", out) } if !strings.Contains(out, "run.id=run-12345") || !strings.Contains(out, "run.status=pending") { t.Errorf("output %q missing start run id/status", out) } if api.lastImportDailyBarsReq() == nil { t.Error("server did not receive an import request before start") } if api.lastStartBacktestReq() == nil { t.Error("server did not receive a start backtest request") } } func TestRunBacktestPollingStopsOnSucceeded(t *testing.T) { var mu sync.Mutex pollCount := 0 api := &fakeAPI{ getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { mu.Lock() defer mu.Unlock() pollCount++ status := altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING if pollCount >= 3 { status = altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED } return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{ Id: req.GetRunId(), Status: status, }, }, nil }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_run_polling", Steps: []Step{ { ID: "poll", Action: ActionPollBacktestRun, Request: Request{ RunID: "run-12345", PollingInterval: Duration(50 * time.Millisecond), }, Expect: Expect{ Status: "ok", RunStatus: "succeeded", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } if !strings.Contains(out, "run.status=succeeded") { t.Errorf("output %q missing run.status=succeeded", out) } mu.Lock() finalPollCount := pollCount mu.Unlock() if finalPollCount < 3 { t.Errorf("pollCount = %d, want >= 3", finalPollCount) } } func TestRunBacktestResultSummaryOutputsMetrics(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Result: &altv1.BacktestResult{ RunId: "run-12345", Summary: &altv1.BacktestSummaryMetrics{ StartingCash: &altv1.Price{Amount: &altv1.Decimal{Value: "10000000"}}, EndingEquity: &altv1.Price{Amount: &altv1.Decimal{Value: "12500000"}}, TotalReturn: &altv1.Decimal{Value: "0.25"}, TradeCount: 42, }, }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_result_summary", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "run-12345", }, Expect: Expect{ Status: "ok", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } for _, want := range []string{ "run.id=run-12345", "starting_cash=10000000", "ending_equity=12500000", "total_return=0.25", "trade_count=42", } { if !strings.Contains(out, want) { t.Errorf("output %q missing %q", out, want) } } } func TestRunBacktestMissingRunIDTypedError(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Error: &altv1.ErrorInfo{ Code: "not_found", Message: "missing run id", }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_missing_run_id", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "nonexistent", }, Expect: Expect{ Status: "error", ErrorCode: "not_found", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } if !strings.Contains(out, "error.code=not_found") { t.Errorf("output %q missing error.code=not_found", out) } } // matrixScenario builds a minimal 1-universe, 1-strategy, 1-timeframe scenario // with the given period list, so matrix batch tests share the same setup. func matrixScenario(name string, periods []MatrixPeriod) *Scenario { return &Scenario{ Name: name, Universes: []UniverseConfig{ {Name: "u1", Provider: "kis", SelectorKind: "watchlist", Market: "kr", Symbols: []string{"005930"}}, }, Matrix: &BacktestScenarioMatrix{ Universes: []string{"u1"}, Strategies: []string{"strat1"}, Timeframes: []string{"daily"}, Periods: periods, }, } } func TestRunBacktestMatrixRunsSequentially(t *testing.T) { var mu sync.Mutex var startCalls int api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { mu.Lock() startCalls++ n := startCalls mu.Unlock() return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{ Id: fmt.Sprintf("run-%d", n), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING, }, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{ Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED, }, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_sequential", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, {ID: "p2", FromUnixMs: 3000, ToUnixMs: 4000}, }) out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } mu.Lock() finalCalls := startCalls mu.Unlock() if finalCalls != 2 { t.Errorf("StartBacktest called %d times, want 2", finalCalls) } for _, want := range []string{ "matrix_run_id=u1__strat1__daily__p1", "matrix_run_id=u1__strat1__daily__p2", "universe=u1", "strategy_id=strat1", "timeframe=daily", "period_id=p1", "period_id=p2", "run.id=run-1", "run.id=run-2", "run.status=succeeded", "steps=2", "passed=2", } { if !strings.Contains(out, want) { t.Errorf("output missing %q", want) } } } func TestRunBacktestMatrixBatchOutputJSONLKeys(t *testing.T) { api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-j1", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED}, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_jsonl", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, }) var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 2 * time.Second, Format: OutputJSONL, }, &stdout) if code != codeOK { t.Fatalf("exit code = %d, want 0", code) } out := stdout.String() for _, want := range []string{`"matrix_run_id"`, `"period_id"`, `"strategy_id"`, `"timeframe"`} { if !strings.Contains(out, want) { t.Errorf("JSONL output missing key %q", want) } } } func TestRunBacktestMatrixReportsTerminalFailedAsLifecycleResult(t *testing.T) { api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-fail", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_FAILED}, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_terminal_failed", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, }) out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (terminal failed is lifecycle evidence) (out=%q)", code, out) } if !strings.Contains(out, "run.status=failed") { t.Errorf("output %q missing run.status=failed", out) } if !strings.Contains(out, "passed=1") { t.Errorf("output %q missing passed=1", out) } } func TestRunBacktestMatrixPollingTimeoutExitsTransport(t *testing.T) { api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-hang", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunResp: &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{ Id: "run-hang", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING, }, }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_polling_timeout", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, }) var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 80 * time.Millisecond, Format: OutputText, }, &stdout) if code != codeTransport { t.Fatalf("exit code = %d, want 3 (codeTransport) (out=%q)", code, stdout.String()) } out := stdout.String() if !strings.Contains(out, "polling timed out") { t.Errorf("output %q missing polling timeout message", out) } if !strings.Contains(out, "status=transport_error") { t.Errorf("output %q missing status=transport_error", out) } } func TestRunBacktestMatrixStartRequestMaterializes(t *testing.T) { var capturedReq *altv1.StartBacktestRequest var mu sync.Mutex api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { mu.Lock() capturedReq = req mu.Unlock() return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-mat", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED}, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_materialize", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1746057600000, ToUnixMs: 1747267200000}, }) _, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0", code) } mu.Lock() req := capturedReq mu.Unlock() if req == nil { t.Fatal("StartBacktest was never called") } spec := req.GetSpec() if spec == nil { t.Fatal("StartBacktest request has nil spec") } if spec.GetMarket() != altv1.Market_MARKET_KR { t.Errorf("spec.Market = %v, want MARKET_KR", spec.GetMarket()) } if spec.GetTimeframe() != altv1.Timeframe_TIMEFRAME_DAILY { t.Errorf("spec.Timeframe = %v, want TIMEFRAME_DAILY", spec.GetTimeframe()) } if spec.GetFromUnixMs() != 1746057600000 { t.Errorf("spec.FromUnixMs = %d, want 1746057600000", spec.GetFromUnixMs()) } if spec.GetToUnixMs() != 1747267200000 { t.Errorf("spec.ToUnixMs = %d, want 1747267200000", spec.GetToUnixMs()) } sel := spec.GetSelector() if sel == nil { t.Fatal("spec.Selector is nil") } if len(sel.GetSymbols()) != 1 || sel.GetSymbols()[0] != "005930" { t.Errorf("spec.Selector.Symbols = %v, want [005930]", sel.GetSymbols()) } } func TestRunBacktestPollingHonorsPollingTimeout(t *testing.T) { // API will keep returning RUNNING, never terminal, so polling should hit polling_timeout api := &fakeAPI{ getBacktestRunResp: &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{ Id: "run-12345", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING, }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_run_polling_timeout", Steps: []Step{ { ID: "poll", Action: ActionPollBacktestRun, Request: Request{ RunID: "run-12345", PollingInterval: Duration(5 * time.Millisecond), PollingTimeout: Duration(25 * time.Millisecond), }, Expect: Expect{ Status: "transport_error", // expected to time out }, }, }, } out, code := runScenario(t, sc, url) if code != codeTransport { t.Fatalf("exit code = %d, want 3 (out=%q)", code, out) } if !strings.Contains(out, "status=transport_error") || !strings.Contains(out, "polling timed out") { t.Errorf("output %q missing polling timeout markers", out) } } func TestRunBacktestMatrixReportsMismatchOnExpectedStatusDifference(t *testing.T) { api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-mismatch", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_FAILED}, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_mismatch", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, }) sc.Matrix.Expect.RunStatus = "succeeded" var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 2 * time.Second, Format: OutputJSONL, }, &stdout) if code != codeAppError { t.Fatalf("exit code = %d, want 1 (codeAppError) (out=%q)", code, stdout.String()) } out := stdout.String() if !strings.Contains(out, `"status":"mismatch"`) { t.Errorf("output %q missing mismatch status in step", out) } if !strings.Contains(out, `expectation mismatch: expected succeeded but got failed`) { t.Errorf("output %q missing expectation mismatch message", out) } if !strings.Contains(out, `"status":"failed"`) { t.Errorf("output %q missing failed status in summary", out) } } func TestRunBacktestMatrixTimeoutPreservesLastNonTerminalStatus(t *testing.T) { api := &fakeAPI{ startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) { return &altv1.StartBacktestResponse{ Run: &altv1.BacktestRun{Id: "run-timeout", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING}, }, nil }, getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) { return &altv1.GetBacktestRunResponse{ Run: &altv1.BacktestRun{ Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING, }, }, nil }, } url := startFakeAPIServer(t, api) sc := matrixScenario("matrix_timeout", []MatrixPeriod{ {ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000}, }) var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 80 * time.Millisecond, Format: OutputJSONL, }, &stdout) if code != codeTransport { t.Fatalf("exit code = %d, want 3 (codeTransport) (out=%q)", code, stdout.String()) } out := stdout.String() if !strings.Contains(out, `"run_status":"running"`) { t.Errorf("output %q missing preserved last non-terminal status \"running\"", out) } if !strings.Contains(out, `"status":"transport_error"`) { t.Errorf("output %q missing transport_error status", out) } } func TestRunBacktestResultSummaryOutputsZeroTradeCount(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Result: &altv1.BacktestResult{ RunId: "run-zero-trade", Summary: &altv1.BacktestSummaryMetrics{ StartingCash: &altv1.Price{Amount: &altv1.Decimal{Value: "10000000"}}, EndingEquity: &altv1.Price{Amount: &altv1.Decimal{Value: "10000000"}}, TotalReturn: &altv1.Decimal{Value: "0.0"}, TradeCount: 0, }, }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "backtest_result_summary_zero", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "run-zero-trade", }, Expect: Expect{ Status: "ok", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } for _, want := range []string{ "run.id=run-zero-trade", "starting_cash=10000000", "ending_equity=10000000", "total_return=0.0", "trade_count=0", } { if !strings.Contains(out, want) { t.Errorf("output %q missing %q", out, want) } } }