alt/apps/cli/internal/operator/runner_error_test.go

package operator

import (
	"bytes"
	"context"
	"strings"
	"testing"
	"time"

	altv1 "git.toki-labs.com/toki/alt/packages/contracts/gen/go/alt/v1"
)

func TestExpectedTypedErrorExitsZero(t *testing.T) {
	api := &fakeAPI{
		getBacktestResultResp: &altv1.GetBacktestResultResponse{
			Error: &altv1.ErrorInfo{
				Code:    "invalid_request",
				Message: "bad run id",
			},
		},
	}
	url := startFakeAPIServer(t, api)
	sc := &Scenario{
		Name: "expected_typed_error",
		Steps: []Step{
			{
				ID:     "result",
				Action: ActionGetBacktestResult,
				Request: Request{
					RunID: "invalid-id",
				},
				Expect: Expect{
					Status:    "error",
					ErrorCode: "invalid_request",
				},
			},
		},
	}

	out, code := runScenario(t, sc, url)
	if code != codeOK {
		t.Fatalf("exit code = %d, want 0 (out=%q)", code, out)
	}
	if !strings.Contains(out, "status=ok") || !strings.Contains(out, "error.code=invalid_request") {
		t.Errorf("output %q missing expected error markers", out)
	}
}

func TestUnexpectedTypedErrorExitsOne(t *testing.T) {
	api := &fakeAPI{
		getBacktestResultResp: &altv1.GetBacktestResultResponse{
			Error: &altv1.ErrorInfo{
				Code:    "invalid_request",
				Message: "bad run id",
			},
		},
	}
	url := startFakeAPIServer(t, api)
	sc := &Scenario{
		Name: "unexpected_typed_error",
		Steps: []Step{
			{
				ID:     "result",
				Action: ActionGetBacktestResult,
				Request: Request{
					RunID: "invalid-id",
				},
				Expect: Expect{
					Status: "ok",
				},
			},
		},
	}

	out, code := runScenario(t, sc, url)
	if code != codeAppError {
		t.Fatalf("exit code = %d, want 1 (out=%q)", code, out)
	}
	if !strings.Contains(out, "status=error") || !strings.Contains(out, "error.code=invalid_request") {
		t.Errorf("output %q missing unexpected error markers", out)
	}
}

func TestTransportFailureExitsThree(t *testing.T) {
	port := freePort(t)
	url := "ws://127.0.0.1:" + itoa(port) + "/socket"
	sc := &Scenario{
		Name:  "transport_failure",
		Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}},
	}

	out, code := runScenario(t, sc, url)
	if code != codeTransport {
		t.Fatalf("exit code = %d, want 3 (out=%q)", code, out)
	}
	if !strings.Contains(out, "status=transport_error") {
		t.Errorf("output %q missing status=transport_error", out)
	}
}

func TestExpectationMismatchExitsOne(t *testing.T) {
	api := &fakeAPI{
		getBacktestResultResp: &altv1.GetBacktestResultResponse{
			Result: &altv1.BacktestResult{
				RunId: "run-123",
			},
		},
	}
	url := startFakeAPIServer(t, api)
	sc := &Scenario{
		Name: "expectation_mismatch",
		Steps: []Step{
			{
				ID:     "result",
				Action: ActionGetBacktestResult,
				Request: Request{
					RunID: "run-123",
				},
				Expect: Expect{
					Status: "error", // expects error but succeeds
				},
			},
		},
	}

	out, code := runScenario(t, sc, url)
	if code != codeAppError {
		t.Fatalf("exit code = %d, want 1 (out=%q)", code, out)
	}
	if !strings.Contains(out, "status=mismatch") {
		t.Errorf("output %q missing status=mismatch", out)
	}
}

func TestJSONLSummaryIncludesExitCode(t *testing.T) {
	// Success path
	{
		api := &fakeAPI{helloCaps: []string{"hello"}}
		url := startFakeAPIServer(t, api)
		sc := &Scenario{
			Name:  "jsonl_exit_code",
			Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}},
		}

		var stdout bytes.Buffer
		code := RunScenario(context.Background(), sc, RunOptions{
			APIURL:  url,
			Timeout: 2 * time.Second,
			Format:  OutputJSONL,
		}, &stdout)

		if code != codeOK {
			t.Fatalf("exit code = %d, want 0", code)
		}

		out := stdout.String()
		if !strings.Contains(out, `"type":"summary"`) {
			t.Fatalf("missing summary in JSONL output")
		}
		if !strings.Contains(out, `"exit_code":0`) {
			t.Errorf("output %q missing \"exit_code\":0 in summary", out)
		}
		if strings.Contains(out, "instrument_count") {
			t.Errorf("output %q should not contain instrument_count for hello action", out)
		}
		if strings.Contains(out, "bar_count") {
			t.Errorf("output %q should not contain bar_count for hello action", out)
		}
	}

	// Failure path (transport failure)
	{
		port := freePort(t)
		badURL := "ws://127.0.0.1:" + itoa(port) + "/socket"
		scFail := &Scenario{
			Name:  "jsonl_exit_code_fail",
			Steps: []Step{{ID: "hello", Action: ActionHello, Expect: Expect{Status: "ok"}}},
		}

		var stdoutFail bytes.Buffer
		codeFail := RunScenario(context.Background(), scFail, RunOptions{
			APIURL:  badURL,
			Timeout: 2 * time.Second,
			Format:  OutputJSONL,
		}, &stdoutFail)

		if codeFail != codeTransport {
			t.Fatalf("exit code = %d, want 3", codeFail)
		}

		outFail := stdoutFail.String()
		if !strings.Contains(outFail, `"type":"summary"`) {
			t.Fatalf("missing summary in JSONL output")
		}
		if !strings.Contains(outFail, `"exit_code":3`) {
			t.Errorf("output %q missing \"exit_code\":3 in summary", outFail)
		}
	}
}

func TestInvalidRequestMatrixOutputsExpectedErrors(t *testing.T) {
	api := &fakeAPI{
		getBacktestResultResp: &altv1.GetBacktestResultResponse{
			Error: &altv1.ErrorInfo{
				Code:    "invalid_request",
				Message: "invalid run id",
			},
		},
		compareBacktestRunsResp: &altv1.CompareBacktestRunsResponse{
			Error: &altv1.ErrorInfo{
				Code:    "not_found",
				Message: "run not found",
			},
		},
		startBacktestResp: &altv1.StartBacktestResponse{
			Error: &altv1.ErrorInfo{
				Code:    "not_found",
				Message: "strategy not found",
			},
		},
	}
	url := startFakeAPIServer(t, api)

	sc, err := LoadScenario(fixture(t, "invalid_request_matrix.yaml"))
	if err != nil {
		t.Fatalf("load scenario: %v", err)
	}

	var stdout bytes.Buffer
	code := RunScenario(context.Background(), sc, RunOptions{
		APIURL:  url,
		Timeout: 2 * time.Second,
		Format:  OutputJSONL,
	}, &stdout)

	if code != codeOK {
		t.Fatalf("exit code = %d, want 0 (out=%q)", code, stdout.String())
	}

	out := stdout.String()
	if !strings.Contains(out, `"type":"summary"`) {
		t.Fatalf("missing summary in JSONL output")
	}
	if !strings.Contains(out, `"exit_code":0`) {
		t.Errorf("output %q missing \"exit_code\":0 in summary", out)
	}

	// Assert step IDs and their error codes in the step output events
	lines := strings.Split(out, "\n")
	foundInvalidRunID := false
	foundCompareMissingRuns := false
	foundStartBacktestMissingStrategy := false

	for _, line := range lines {
		if strings.Contains(line, `"step":"invalid_run_id"`) {
			foundInvalidRunID = true
			if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"invalid_request"`) {
				t.Errorf("invalid_run_id event line %q missing expected status or error_code", line)
			}
		}
		if strings.Contains(line, `"step":"compare_missing_runs"`) {
			foundCompareMissingRuns = true
			if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"not_found"`) {
				t.Errorf("compare_missing_runs event line %q missing expected status or error_code", line)
			}
		}
		if strings.Contains(line, `"step":"start_backtest_missing_strategy"`) {
			foundStartBacktestMissingStrategy = true
			if !strings.Contains(line, `"status":"ok"`) || !strings.Contains(line, `"error_code":"not_found"`) {
				t.Errorf("start_backtest_missing_strategy event line %q missing expected status or error_code", line)
			}
		}
	}

	if !foundInvalidRunID {
		t.Error("missing invalid_run_id step event in output")
	}
	if !foundCompareMissingRuns {
		t.Error("missing compare_missing_runs step event in output")
	}
	if !foundStartBacktestMissingStrategy {
		t.Error("missing start_backtest_missing_strategy step event in output")
	}
}