package operator import ( "bytes" "context" "strings" "testing" "time" altv1 "git.toki-labs.com/toki/alt/packages/contracts/gen/go/alt/v1" ) func TestExpectedTypedErrorExitsZero(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Error: &altv1.ErrorInfo{ Code: "invalid_request", Message: "bad run id", }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "expected_typed_error", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "invalid-id", }, Expect: Expect{ Status: "error", ErrorCode: "invalid_request", }, }, }, } out, code := runScenario(t, sc, url) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, out) } if !strings.Contains(out, "status=ok") || !strings.Contains(out, "error.code=invalid_request") { t.Errorf("output %q missing expected error markers", out) } } func TestUnexpectedTypedErrorExitsOne(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Error: &altv1.ErrorInfo{ Code: "invalid_request", Message: "bad run id", }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "unexpected_typed_error", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "invalid-id", }, Expect: Expect{ Status: "ok", }, }, }, } out, code := runScenario(t, sc, url) if code != codeAppError { t.Fatalf("exit code = %d, want 1 (out=%q)", code, out) } if !strings.Contains(out, "status=error") || !strings.Contains(out, "error.code=invalid_request") { t.Errorf("output %q missing unexpected error markers", out) } } func TestTransportFailureExitsThree(t *testing.T) { port := freePort(t) url := "ws://127.0.0.1:" + itoa(port) + "/socket" sc := &Scenario{ Name: "transport_failure", Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}}, } out, code := runScenario(t, sc, url) if code != codeTransport { t.Fatalf("exit code = %d, want 3 (out=%q)", code, out) } if !strings.Contains(out, "status=transport_error") { t.Errorf("output %q missing status=transport_error", out) } } func TestExpectationMismatchExitsOne(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Result: &altv1.BacktestResult{ RunId: "run-123", }, }, } url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "expectation_mismatch", Steps: []Step{ { ID: "result", Action: ActionGetBacktestResult, Request: Request{ RunID: "run-123", }, Expect: Expect{ Status: "error", // expects error but succeeds }, }, }, } out, code := runScenario(t, sc, url) if code != codeAppError { t.Fatalf("exit code = %d, want 1 (out=%q)", code, out) } if !strings.Contains(out, "status=mismatch") { t.Errorf("output %q missing status=mismatch", out) } } func TestJSONLSummaryIncludesExitCode(t *testing.T) { // Success path { api := &fakeAPI{helloCaps: []string{"hello"}} url := startFakeAPIServer(t, api) sc := &Scenario{ Name: "jsonl_exit_code", Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}}, } var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 2 * time.Second, Format: OutputJSONL, }, &stdout) if code != codeOK { t.Fatalf("exit code = %d, want 0", code) } out := stdout.String() if !strings.Contains(out, `"type":"summary"`) { t.Fatalf("missing summary in JSONL output") } if !strings.Contains(out, `"exit_code":0`) { t.Errorf("output %q missing \"exit_code\":0 in summary", out) } if strings.Contains(out, "instrument_count") { t.Errorf("output %q should not contain instrument_count for hello action", out) } if strings.Contains(out, "bar_count") { t.Errorf("output %q should not contain bar_count for hello action", out) } } // Failure path (transport failure) { port := freePort(t) badURL := "ws://127.0.0.1:" + itoa(port) + "/socket" scFail := &Scenario{ Name: "jsonl_exit_code_fail", Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}}, } var stdoutFail bytes.Buffer codeFail := RunScenario(context.Background(), scFail, RunOptions{ APIURL: badURL, Timeout: 2 * time.Second, Format: OutputJSONL, }, &stdoutFail) if codeFail != codeTransport { t.Fatalf("exit code = %d, want 3", codeFail) } outFail := stdoutFail.String() if !strings.Contains(outFail, `"type":"summary"`) { t.Fatalf("missing summary in JSONL output") } if !strings.Contains(outFail, `"exit_code":3`) { t.Errorf("output %q missing \"exit_code\":3 in summary", outFail) } } } func TestInvalidRequestMatrixOutputsExpectedErrors(t *testing.T) { api := &fakeAPI{ getBacktestResultResp: &altv1.GetBacktestResultResponse{ Error: &altv1.ErrorInfo{ Code: "invalid_request", Message: "invalid run id", }, }, compareBacktestRunsResp: &altv1.CompareBacktestRunsResponse{ Error: &altv1.ErrorInfo{ Code: "not_found", Message: "run not found", }, }, startBacktestResp: &altv1.StartBacktestResponse{ Error: &altv1.ErrorInfo{ Code: "not_found", Message: "strategy not found", }, }, } url := startFakeAPIServer(t, api) sc, err := LoadScenario(fixture(t, "invalid_request_matrix.yaml")) if err != nil { t.Fatalf("load scenario: %v", err) } var stdout bytes.Buffer code := RunScenario(context.Background(), sc, RunOptions{ APIURL: url, Timeout: 2 * time.Second, Format: OutputJSONL, }, &stdout) if code != codeOK { t.Fatalf("exit code = %d, want 0 (out=%q)", code, stdout.String()) } out := stdout.String() if !strings.Contains(out, `"type":"summary"`) { t.Fatalf("missing summary in JSONL output") } if !strings.Contains(out, `"exit_code":0`) { t.Errorf("output %q missing \"exit_code\":0 in summary", out) } // Assert step IDs and their error codes in the step output events lines := strings.Split(out, "\n") foundInvalidRunID := false foundCompareMissingRuns := false foundStartBacktestMissingStrategy := false for _, line := range lines { if strings.Contains(line, `"step":"invalid_run_id"`) { foundInvalidRunID = true if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"invalid_request"`) { t.Errorf("invalid_run_id event line %q missing expected status or error_code", line) } } if strings.Contains(line, `"step":"compare_missing_runs"`) { foundCompareMissingRuns = true if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"not_found"`) { t.Errorf("compare_missing_runs event line %q missing expected status or error_code", line) } } if strings.Contains(line, `"step":"start_backtest_missing_strategy"`) { foundStartBacktestMissingStrategy = true if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"not_found"`) { t.Errorf("start_backtest_missing_strategy event line %q missing expected status or error_code", line) } } } if !foundInvalidRunID { t.Error("missing invalid_run_id step event in output") } if !foundCompareMissingRuns { t.Error("missing compare_missing_runs step event in output") } if !foundStartBacktestMissingStrategy { t.Error("missing start_backtest_missing_strategy step event in output") } }