toki
243a1293f4
- Add backtest matrix test cases (success, failed, mismatch, timeout) - Update runner with backtest scenario support - Update scenario.go with handoff validation - Update handoff tests for backtest scenarios - Remove obsolete plan/review documents (moved to archive)
642 lines
18 KiB
Go
642 lines
18 KiB
Go
package operator
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
altv1 "git.toki-labs.com/toki/alt/packages/contracts/gen/go/alt/v1"
|
|
)
|
|
|
|
func TestRunBacktestRequestOutputsRunID(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestResp: &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: "run-12345",
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING,
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_run_request",
|
|
Steps: []Step{
|
|
{
|
|
ID: "start",
|
|
Action: ActionStartBacktest,
|
|
Request: Request{
|
|
StrategyID: "strategy-v1",
|
|
Market: "kr",
|
|
Timeframe: "daily",
|
|
FromUnixMs: 1746057600000,
|
|
ToUnixMs: 1747267200000,
|
|
Symbols: []string{"005930"},
|
|
},
|
|
Expect: Expect{
|
|
Status: "ok",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
if !strings.Contains(out, "run.id=run-12345") {
|
|
t.Errorf("output %q missing run.id=run-12345", out)
|
|
}
|
|
if !strings.Contains(out, "run.status=pending") {
|
|
t.Errorf("output %q missing run.status=pending", out)
|
|
}
|
|
|
|
got := api.lastStartBacktestReq()
|
|
if got == nil {
|
|
t.Fatal("server did not receive a start backtest request")
|
|
}
|
|
if got.GetSpec().GetStrategyId() != "strategy-v1" {
|
|
t.Errorf("server received strategy_id = %q, want strategy-v1", got.GetSpec().GetStrategyId())
|
|
}
|
|
if len(got.GetSpec().GetSelector().GetSymbols()) != 1 || got.GetSpec().GetSelector().GetSymbols()[0] != "005930" {
|
|
t.Errorf("server received symbols = %+v, want [005930]", got.GetSpec().GetSelector().GetSymbols())
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestImportThenStartFlow(t *testing.T) {
|
|
api := &fakeAPI{
|
|
importDailyBarsResp: &altv1.ImportDailyBarsResponse{Provider: "kis", InstrumentCount: 1, BarCount: 42},
|
|
startBacktestResp: &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: "run-12345",
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING,
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_run_request",
|
|
Steps: []Step{
|
|
{
|
|
ID: "import_kr_daily_bars",
|
|
Action: ActionImportDailyBars,
|
|
Request: Request{
|
|
Provider: "kis",
|
|
SelectorKind: "watchlist",
|
|
Market: "kr",
|
|
Venue: "krx",
|
|
Symbols: []string{"005930"},
|
|
FromYYYYMMDD: "20240527",
|
|
ToYYYYMMDD: "20240528",
|
|
},
|
|
Expect: Expect{Status: "ok"},
|
|
},
|
|
{
|
|
ID: "start_run",
|
|
Action: ActionStartBacktest,
|
|
Request: Request{
|
|
StrategyID: "strategy-v1",
|
|
Market: "kr",
|
|
Timeframe: "daily",
|
|
FromUnixMs: 1746057600000,
|
|
ToUnixMs: 1747267200000,
|
|
Symbols: []string{"005930"},
|
|
},
|
|
Expect: Expect{Status: "ok"},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
// The import step runs before the start step and reports imported counts.
|
|
if !strings.Contains(out, "instrument_count=1") || !strings.Contains(out, "bar_count=42") {
|
|
t.Errorf("output %q missing import counts", out)
|
|
}
|
|
if !strings.Contains(out, "run.id=run-12345") || !strings.Contains(out, "run.status=pending") {
|
|
t.Errorf("output %q missing start run id/status", out)
|
|
}
|
|
if api.lastImportDailyBarsReq() == nil {
|
|
t.Error("server did not receive an import request before start")
|
|
}
|
|
if api.lastStartBacktestReq() == nil {
|
|
t.Error("server did not receive a start backtest request")
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestPollingStopsOnSucceeded(t *testing.T) {
|
|
var mu sync.Mutex
|
|
pollCount := 0
|
|
|
|
api := &fakeAPI{
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
pollCount++
|
|
status := altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING
|
|
if pollCount >= 3 {
|
|
status = altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED
|
|
}
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: req.GetRunId(),
|
|
Status: status,
|
|
},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_run_polling",
|
|
Steps: []Step{
|
|
{
|
|
ID: "poll",
|
|
Action: ActionPollBacktestRun,
|
|
Request: Request{
|
|
RunID: "run-12345",
|
|
PollingInterval: Duration(50 * time.Millisecond),
|
|
},
|
|
Expect: Expect{
|
|
Status: "ok",
|
|
RunStatus: "succeeded",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
if !strings.Contains(out, "run.status=succeeded") {
|
|
t.Errorf("output %q missing run.status=succeeded", out)
|
|
}
|
|
|
|
mu.Lock()
|
|
finalPollCount := pollCount
|
|
mu.Unlock()
|
|
if finalPollCount < 3 {
|
|
t.Errorf("pollCount = %d, want >= 3", finalPollCount)
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestResultSummaryOutputsMetrics(t *testing.T) {
|
|
api := &fakeAPI{
|
|
getBacktestResultResp: &altv1.GetBacktestResultResponse{
|
|
Result: &altv1.BacktestResult{
|
|
RunId: "run-12345",
|
|
Summary: &altv1.BacktestSummaryMetrics{
|
|
StartingCash: &altv1.Price{Amount: &altv1.Decimal{Value: "10000000"}},
|
|
EndingEquity: &altv1.Price{Amount: &altv1.Decimal{Value: "12500000"}},
|
|
TotalReturn: &altv1.Decimal{Value: "0.25"},
|
|
TradeCount: 42,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_result_summary",
|
|
Steps: []Step{
|
|
{
|
|
ID: "result",
|
|
Action: ActionGetBacktestResult,
|
|
Request: Request{
|
|
RunID: "run-12345",
|
|
},
|
|
Expect: Expect{
|
|
Status: "ok",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
for _, want := range []string{
|
|
"run.id=run-12345",
|
|
"starting_cash=10000000",
|
|
"ending_equity=12500000",
|
|
"total_return=0.25",
|
|
"trade_count=42",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("output %q missing %q", out, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMissingRunIDTypedError(t *testing.T) {
|
|
api := &fakeAPI{
|
|
getBacktestResultResp: &altv1.GetBacktestResultResponse{
|
|
Error: &altv1.ErrorInfo{
|
|
Code: "not_found",
|
|
Message: "missing run id",
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_missing_run_id",
|
|
Steps: []Step{
|
|
{
|
|
ID: "result",
|
|
Action: ActionGetBacktestResult,
|
|
Request: Request{
|
|
RunID: "nonexistent",
|
|
},
|
|
Expect: Expect{
|
|
Status: "error",
|
|
ErrorCode: "not_found",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
if !strings.Contains(out, "error.code=not_found") {
|
|
t.Errorf("output %q missing error.code=not_found", out)
|
|
}
|
|
}
|
|
|
|
// matrixScenario builds a minimal 1-universe, 1-strategy, 1-timeframe scenario
|
|
// with the given period list, so matrix batch tests share the same setup.
|
|
func matrixScenario(name string, periods []MatrixPeriod) *Scenario {
|
|
return &Scenario{
|
|
Name: name,
|
|
Universes: []UniverseConfig{
|
|
{Name: "u1", Provider: "kis", SelectorKind: "watchlist", Market: "kr", Symbols: []string{"005930"}},
|
|
},
|
|
Matrix: &BacktestScenarioMatrix{
|
|
Universes: []string{"u1"},
|
|
Strategies: []string{"strat1"},
|
|
Timeframes: []string{"daily"},
|
|
Periods: periods,
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixRunsSequentially(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var startCalls int
|
|
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
mu.Lock()
|
|
startCalls++
|
|
n := startCalls
|
|
mu.Unlock()
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: fmt.Sprintf("run-%d", n),
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING,
|
|
},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: req.GetRunId(),
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED,
|
|
},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_sequential", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
{ID: "p2", FromUnixMs: 3000, ToUnixMs: 4000},
|
|
})
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
|
|
}
|
|
|
|
mu.Lock()
|
|
finalCalls := startCalls
|
|
mu.Unlock()
|
|
if finalCalls != 2 {
|
|
t.Errorf("StartBacktest called %d times, want 2", finalCalls)
|
|
}
|
|
|
|
for _, want := range []string{
|
|
"matrix_run_id=u1__strat1__daily__p1",
|
|
"matrix_run_id=u1__strat1__daily__p2",
|
|
"universe=u1",
|
|
"strategy_id=strat1",
|
|
"timeframe=daily",
|
|
"period_id=p1",
|
|
"period_id=p2",
|
|
"run.id=run-1",
|
|
"run.id=run-2",
|
|
"run.status=succeeded",
|
|
"steps=2",
|
|
"passed=2",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("output missing %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixBatchOutputJSONLKeys(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-j1", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_jsonl", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
})
|
|
|
|
var stdout bytes.Buffer
|
|
code := RunScenario(context.Background(), sc, RunOptions{
|
|
APIURL: url,
|
|
Timeout: 2 * time.Second,
|
|
Format: OutputJSONL,
|
|
}, &stdout)
|
|
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0", code)
|
|
}
|
|
|
|
out := stdout.String()
|
|
for _, want := range []string{`"matrix_run_id"`, `"period_id"`, `"strategy_id"`, `"timeframe"`} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("JSONL output missing key %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixReportsTerminalFailedAsLifecycleResult(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-fail", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_FAILED},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_terminal_failed", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
})
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0 (terminal failed is lifecycle evidence) (out=%q)", code, out)
|
|
}
|
|
if !strings.Contains(out, "run.status=failed") {
|
|
t.Errorf("output %q missing run.status=failed", out)
|
|
}
|
|
if !strings.Contains(out, "passed=1") {
|
|
t.Errorf("output %q missing passed=1", out)
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixPollingTimeoutExitsTransport(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-hang", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunResp: &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: "run-hang",
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING,
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_polling_timeout", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
})
|
|
|
|
var stdout bytes.Buffer
|
|
code := RunScenario(context.Background(), sc, RunOptions{
|
|
APIURL: url,
|
|
Timeout: 80 * time.Millisecond,
|
|
Format: OutputText,
|
|
}, &stdout)
|
|
|
|
if code != codeTransport {
|
|
t.Fatalf("exit code = %d, want 3 (codeTransport) (out=%q)", code, stdout.String())
|
|
}
|
|
out := stdout.String()
|
|
if !strings.Contains(out, "polling timed out") {
|
|
t.Errorf("output %q missing polling timeout message", out)
|
|
}
|
|
if !strings.Contains(out, "status=transport_error") {
|
|
t.Errorf("output %q missing status=transport_error", out)
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixStartRequestMaterializes(t *testing.T) {
|
|
var capturedReq *altv1.StartBacktestRequest
|
|
var mu sync.Mutex
|
|
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
mu.Lock()
|
|
capturedReq = req
|
|
mu.Unlock()
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-mat", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_SUCCEEDED},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_materialize", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1746057600000, ToUnixMs: 1747267200000},
|
|
})
|
|
|
|
_, code := runScenario(t, sc, url)
|
|
if code != codeOK {
|
|
t.Fatalf("exit code = %d, want 0", code)
|
|
}
|
|
|
|
mu.Lock()
|
|
req := capturedReq
|
|
mu.Unlock()
|
|
if req == nil {
|
|
t.Fatal("StartBacktest was never called")
|
|
}
|
|
spec := req.GetSpec()
|
|
if spec == nil {
|
|
t.Fatal("StartBacktest request has nil spec")
|
|
}
|
|
if spec.GetMarket() != altv1.Market_MARKET_KR {
|
|
t.Errorf("spec.Market = %v, want MARKET_KR", spec.GetMarket())
|
|
}
|
|
if spec.GetTimeframe() != altv1.Timeframe_TIMEFRAME_DAILY {
|
|
t.Errorf("spec.Timeframe = %v, want TIMEFRAME_DAILY", spec.GetTimeframe())
|
|
}
|
|
if spec.GetFromUnixMs() != 1746057600000 {
|
|
t.Errorf("spec.FromUnixMs = %d, want 1746057600000", spec.GetFromUnixMs())
|
|
}
|
|
if spec.GetToUnixMs() != 1747267200000 {
|
|
t.Errorf("spec.ToUnixMs = %d, want 1747267200000", spec.GetToUnixMs())
|
|
}
|
|
sel := spec.GetSelector()
|
|
if sel == nil {
|
|
t.Fatal("spec.Selector is nil")
|
|
}
|
|
if len(sel.GetSymbols()) != 1 || sel.GetSymbols()[0] != "005930" {
|
|
t.Errorf("spec.Selector.Symbols = %v, want [005930]", sel.GetSymbols())
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestPollingHonorsPollingTimeout(t *testing.T) {
|
|
// API will keep returning RUNNING, never terminal, so polling should hit polling_timeout
|
|
api := &fakeAPI{
|
|
getBacktestRunResp: &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: "run-12345",
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING,
|
|
},
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
sc := &Scenario{
|
|
Name: "backtest_run_polling_timeout",
|
|
Steps: []Step{
|
|
{
|
|
ID: "poll",
|
|
Action: ActionPollBacktestRun,
|
|
Request: Request{
|
|
RunID: "run-12345",
|
|
PollingInterval: Duration(5 * time.Millisecond),
|
|
PollingTimeout: Duration(25 * time.Millisecond),
|
|
},
|
|
Expect: Expect{
|
|
Status: "transport_error", // expected to time out
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
out, code := runScenario(t, sc, url)
|
|
if code != codeTransport {
|
|
t.Fatalf("exit code = %d, want 3 (out=%q)", code, out)
|
|
}
|
|
if !strings.Contains(out, "status=transport_error") || !strings.Contains(out, "polling timed out") {
|
|
t.Errorf("output %q missing polling timeout markers", out)
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixReportsMismatchOnExpectedStatusDifference(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-mismatch", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{Id: req.GetRunId(), Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_FAILED},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_mismatch", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
})
|
|
sc.Matrix.Expect.RunStatus = "succeeded"
|
|
|
|
var stdout bytes.Buffer
|
|
code := RunScenario(context.Background(), sc, RunOptions{
|
|
APIURL: url,
|
|
Timeout: 2 * time.Second,
|
|
Format: OutputJSONL,
|
|
}, &stdout)
|
|
|
|
if code != codeAppError {
|
|
t.Fatalf("exit code = %d, want 1 (codeAppError) (out=%q)", code, stdout.String())
|
|
}
|
|
out := stdout.String()
|
|
if !strings.Contains(out, `"status":"mismatch"`) {
|
|
t.Errorf("output %q missing mismatch status in step", out)
|
|
}
|
|
if !strings.Contains(out, `expectation mismatch: expected succeeded but got failed`) {
|
|
t.Errorf("output %q missing expectation mismatch message", out)
|
|
}
|
|
if !strings.Contains(out, `"status":"failed"`) {
|
|
t.Errorf("output %q missing failed status in summary", out)
|
|
}
|
|
}
|
|
|
|
func TestRunBacktestMatrixTimeoutPreservesLastNonTerminalStatus(t *testing.T) {
|
|
api := &fakeAPI{
|
|
startBacktestFunc: func(req *altv1.StartBacktestRequest) (*altv1.StartBacktestResponse, error) {
|
|
return &altv1.StartBacktestResponse{
|
|
Run: &altv1.BacktestRun{Id: "run-timeout", Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_PENDING},
|
|
}, nil
|
|
},
|
|
getBacktestRunFunc: func(req *altv1.GetBacktestRunRequest) (*altv1.GetBacktestRunResponse, error) {
|
|
return &altv1.GetBacktestRunResponse{
|
|
Run: &altv1.BacktestRun{
|
|
Id: req.GetRunId(),
|
|
Status: altv1.BacktestRunStatus_BACKTEST_RUN_STATUS_RUNNING,
|
|
},
|
|
}, nil
|
|
},
|
|
}
|
|
url := startFakeAPIServer(t, api)
|
|
|
|
sc := matrixScenario("matrix_timeout", []MatrixPeriod{
|
|
{ID: "p1", FromUnixMs: 1000, ToUnixMs: 2000},
|
|
})
|
|
|
|
var stdout bytes.Buffer
|
|
code := RunScenario(context.Background(), sc, RunOptions{
|
|
APIURL: url,
|
|
Timeout: 80 * time.Millisecond,
|
|
Format: OutputJSONL,
|
|
}, &stdout)
|
|
|
|
if code != codeTransport {
|
|
t.Fatalf("exit code = %d, want 3 (codeTransport) (out=%q)", code, stdout.String())
|
|
}
|
|
out := stdout.String()
|
|
if !strings.Contains(out, `"run_status":"running"`) {
|
|
t.Errorf("output %q missing preserved last non-terminal status \"running\"", out)
|
|
}
|
|
if !strings.Contains(out, `"status":"transport_error"`) {
|
|
t.Errorf("output %q missing transport_error status", out)
|
|
}
|
|
}
|