iop/apps/edge/internal/openai/single_request_handler_test.go

package openai

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/testutil"
	dto "github.com/prometheus/client_model/go"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
	"google.golang.org/protobuf/proto"

	"iop/apps/edge/internal/authprojection"
	edgenode "iop/apps/edge/internal/node"
	edgeservice "iop/apps/edge/internal/service"
	"iop/packages/go/config"
	iop "iop/proto/gen/iop"
)

const (
	testSingleRequestModel = "virtual-single-request"
	testSingleRequestToken = "single-request-token"
)

type anthropicSingleRequestExecutorFunc func(context.Context, edgeservice.SingleRequestRequest, edgeservice.SingleRequestController) error

func (f anthropicSingleRequestExecutorFunc) ExecuteSingleRequest(
	ctx context.Context,
	req edgeservice.SingleRequestRequest,
	ctrl edgeservice.SingleRequestController,
) error {
	return f(ctx, req, ctrl)
}

// newAdmittedAnthropicSingleRequestService builds the same catalog and ready
// owner preconditions that the public service requires in production. Endpoint
// tests must cross this boundary rather than using a zero-value Service.
func newAdmittedAnthropicSingleRequestService(
	t *testing.T,
	executor edgeservice.SingleRequestExecutor,
	workspaceRef string,
) *edgeservice.Service {
	t.Helper()
	const nodeID = "workspace-node"

	registry := edgenode.NewRegistry()
	registry.Register(&edgenode.NodeEntry{NodeID: nodeID, Alias: "workspace"})
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:    nodeID,
		Alias: "workspace",
		Token: "workspace-node-token",
		Workspaces: []config.WorkspaceDefinition{{
			Ref:          workspaceRef,
			Platform:     "darwin",
			Root:         "/Users/operator/project",
			Operations:   []config.WorkspaceOperation{config.WorkspaceOpRead},
			MaxReadBytes: 4096,
		}},
	})

	svc := edgeservice.New(registry, nil)
	svc.SetNodeStore(store)
	svc.SetSingleRequestExecutor(executor)
	return svc
}

func newAnthropicSingleRequestServer(t *testing.T, svc runService) *Server {
	t.Helper()
	now := time.Date(2026, 8, 6, 12, 0, 0, 0, time.UTC)
	cache := authprojection.NewCache(authprojection.DefaultLimits(), func() time.Time { return now })
	routes := map[string]authprojection.Route{
		"plan": {
			RouteID: "public-plan", PrincipalRef: "principal-1", CredentialSlotRef: "slot-plan",
			ProfileID: "profile-plan", UpstreamModel: "served-plan", ResourceSelector: "default",
		},
		"work": {
			RouteID: "public-work", PrincipalRef: "principal-1", CredentialSlotRef: "slot-work",
			ProfileID: "profile-work", UpstreamModel: "served-work", ResourceSelector: "default",
		},
		"review": {
			RouteID: "public-review", PrincipalRef: "principal-1", CredentialSlotRef: "slot-review",
			ProfileID: "profile-review", UpstreamModel: "served-review", ResourceSelector: "default",
		},
	}
	if err := cache.Apply(makeTestProjection(
		1, now, time.Hour,
		map[string]string{testSingleRequestToken: "principal-1"}, routes,
	)); err != nil {
		t.Fatalf("apply projection: %v", err)
	}

	deterministicCounter := config.TokenCounterConf{Mode: config.TokenCounterDeterministic}
	srv := NewServer(config.EdgeOpenAIConf{}, svc, nil)
	setManagedPrincipalProjection(srv, cache)
	srv.SetExecutionPresets([]config.ExecutionPreset{validSingleRequestPreset()})
	srv.SetModelCatalog([]config.ModelCatalogEntry{
		{ID: testSingleRequestModel, ExecutionPreset: "preset-single-request"},
		{ID: "plan-model", Providers: map[string]string{"provider-plan": "served-plan"}, TokenCounter: &deterministicCounter},
		{ID: "work-model", Providers: map[string]string{"provider-work": "served-work"}},
		{ID: "review-model", Providers: map[string]string{"provider-review": "served-review"}},
	})
	return srv
}

func newAnthropicSingleRequestHTTPReq(t *testing.T, ctx context.Context, target, path, body string) *http.Request {
	t.Helper()
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, target+path, strings.NewReader(body))
	if err != nil {
		t.Fatalf("new request: %v", err)
	}
	req.Header.Set("Authorization", "Bearer "+testSingleRequestToken)
	req.Header.Set(anthropicVersionHeader, anthropicSupportedVersion)
	req.Header.Set("Content-Type", "application/json")
	return req
}

func serveAnthropicSingleRequest(t *testing.T, srv *Server, ctx context.Context, path, body string, w http.ResponseWriter) {
	t.Helper()
	req := newAnthropicSingleRequestHTTPReq(t, ctx, "http://edge.invalid", path, body)
	srv.routes().ServeHTTP(w, req)
}

func submitAnthropicSingleRequestLifecycle(
	req edgeservice.SingleRequestRequest,
	ctrl edgeservice.SingleRequestController,
	result string,
) error {
	type step struct {
		stage edgeservice.SingleRequestState
		saved edgeservice.SingleRequestState
	}
	steps := []step{
		{stage: edgeservice.SingleRequestStatePlanning},
		{stage: edgeservice.SingleRequestStateWorking},
		{stage: edgeservice.SingleRequestStateReviewing},
		{stage: edgeservice.SingleRequestStateRepairing},
		{stage: edgeservice.SingleRequestStateFinalizing},
	}
	for index, item := range steps {
		envelope := edgeservice.SingleRequestEnvelope{
			RequestID:  req.RequestID,
			Sequence:   uint64(index + 1),
			Stage:      item.stage,
			SavedStage: item.saved,
			Message:    "PRIVATE_STAGE_SENTINEL",
		}
		if item.stage == edgeservice.SingleRequestStateFinalizing {
			envelope.Result = &edgeservice.SingleRequestResult{Output: result}
		}
		if err := ctrl.SubmitEnvelope(envelope); err != nil {
			return err
		}
	}
	return nil
}

func TestAnthropicSingleRequestUsesOnePost(t *testing.T) {
	const (
		privatePrompt = "PRIVATE_CALLER_PROMPT_SENTINEL"
		finalOutput   = "workspace task completed"
	)
	var calls atomic.Int32
	requestCh := make(chan edgeservice.SingleRequestRequest, 1)
	controllerCh := make(chan edgeservice.SingleRequestController, 1)
	executor := anthropicSingleRequestExecutorFunc(func(
		_ context.Context,
		req edgeservice.SingleRequestRequest,
		ctrl edgeservice.SingleRequestController,
	) error {
		calls.Add(1)
		requestCh <- req
		if err := submitAnthropicSingleRequestLifecycle(req, ctrl, finalOutput); err != nil {
			return err
		}
		controllerCh <- ctrl
		return nil
	})
	svc := newAdmittedAnthropicSingleRequestService(t, executor, "ws-opaque-ref")
	srv := newAnthropicSingleRequestServer(t, svc)
	httpServer := httptest.NewServer(srv.routes())
	defer httpServer.Close()

	before := testutil.ToFloat64(singleRequestIngressTotal)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":128,"messages":[{"role":"user","content":"` + privatePrompt + `"}],"tools":[{"name":"caller_tool","input_schema":{"type":"object"}}]}`
	req := newAnthropicSingleRequestHTTPReq(t, ctx, httpServer.URL, "/v1/messages", body)
	response, err := httpServer.Client().Do(req)
	if err != nil {
		t.Fatalf("POST /v1/messages: %v", err)
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		payload, _ := io.ReadAll(response.Body)
		t.Fatalf("status=%d body=%s", response.StatusCode, payload)
	}
	if got := response.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") {
		t.Fatalf("content-type=%q, want buffered application/json terminal", got)
	}

	var terminal anthropicMessageResponse
	decoder := json.NewDecoder(response.Body)
	if err := decoder.Decode(&terminal); err != nil {
		t.Fatalf("decode terminal: %v", err)
	}
	var extra json.RawMessage
	if err := decoder.Decode(&extra); err != io.EOF {
		t.Fatalf("expected exactly one JSON terminal, trailing decode error=%v value=%s", err, extra)
	}
	if terminal.Model != testSingleRequestModel || terminal.Type != "message" || terminal.Role != "assistant" {
		t.Fatalf("public terminal identity mismatch: %+v", terminal)
	}
	if terminal.StopReason == nil || *terminal.StopReason != "end_turn" {
		t.Fatalf("stop_reason=%v, want end_turn", terminal.StopReason)
	}
	if len(terminal.Content) != 1 || terminal.Content[0]["type"] != "text" || terminal.Content[0]["text"] != finalOutput {
		t.Fatalf("terminal content=%+v, want one sanitized text block", terminal.Content)
	}
	encoded, err := json.Marshal(terminal)
	if err != nil {
		t.Fatal(err)
	}
	for _, privateValue := range []string{
		privatePrompt, "PRIVATE_STAGE_SENTINEL", "caller_tool", "tool_use",
		"ws-opaque-ref", "plan-model", "provider-plan", "slot-plan",
	} {
		if strings.Contains(string(encoded), privateValue) {
			t.Fatalf("terminal leaked %q: %s", privateValue, encoded)
		}
	}

	if got := testutil.ToFloat64(singleRequestIngressTotal) - before; got != 1 {
		t.Fatalf("single-request ingress counter delta=%v, want 1", got)
	}
	if got := calls.Load(); got != 1 {
		t.Fatalf("executor calls=%d, want 1", got)
	}
	captured := <-requestCh
	if captured.Binding == nil || captured.Binding.PublicModel != testSingleRequestModel {
		t.Fatalf("captured immutable binding=%+v", captured.Binding)
	}
	if workspace := captured.Binding.Workspace; workspace == nil ||
		workspace.Ref != "ws-opaque-ref" ||
		workspace.NodeID != "workspace-node" ||
		workspace.ConnectionGeneration == 0 ||
		len(workspace.OperationIDs) != 1 || workspace.OperationIDs[0] != string(config.WorkspaceOpRead) {
		t.Fatalf("captured workspace projection=%#v, want frozen ready read capability", workspace)
	}
	if !strings.Contains(captured.Prompt, privatePrompt) {
		t.Fatalf("executor did not receive immutable caller input: %q", captured.Prompt)
	}
	controller := <-controllerCh
	if got := controller.State(); got != edgeservice.SingleRequestStateCompleted {
		t.Fatalf("terminal acknowledgement state=%s, want completed", got)
	}

	families, err := prometheus.DefaultGatherer.Gather()
	if err != nil {
		t.Fatalf("gather metrics: %v", err)
	}
	foundMetric := false
	for _, family := range families {
		if family.GetName() != "iop_anthropic_single_request_ingress_total" {
			continue
		}
		foundMetric = true
		for _, metric := range family.Metric {
			if len(metric.Label) != 0 {
				t.Fatalf("single-request ingress metric has request-derived labels: %+v", metric.Label)
			}
		}
	}
	if !foundMetric {
		t.Fatal("registered single-request ingress metric was not gathered")
	}
}

type anthropicInternalToolExecutor struct {
	results       chan edgeservice.InternalWorkspaceToolResult
	continueCount atomic.Int32
}

func newAnthropicInternalToolExecutor() *anthropicInternalToolExecutor {
	return &anthropicInternalToolExecutor{results: make(chan edgeservice.InternalWorkspaceToolResult, 2)}
}

func (e *anthropicInternalToolExecutor) ExecuteSingleRequest(
	ctx context.Context,
	req edgeservice.SingleRequestRequest,
	ctrl edgeservice.SingleRequestController,
) error {
	sequence := uint64(1)
	if err := ctrl.SubmitEnvelope(edgeservice.SingleRequestEnvelope{RequestID: req.RequestID, Sequence: sequence, Stage: edgeservice.SingleRequestStatePlanning}); err != nil {
		return err
	}
	calls := []*edgeservice.InternalWorkspaceToolCall{
		{
			RequestID: req.RequestID, StageID: "plan", ToolCallID: "tool-read",
			Name: edgeservice.InternalWorkspaceToolRead, Arguments: json.RawMessage(`{"relative_path":"README.md"}`),
		},
		{
			RequestID: req.RequestID, StageID: "plan", ToolCallID: "tool-write",
			Name:      edgeservice.InternalWorkspaceToolWrite,
			Arguments: json.RawMessage(`{"relative_path":"result.txt","content":"PRIVATE_INTERNAL_ARGUMENT_SENTINEL"}`),
		},
	}
	for _, call := range calls {
		sequence++
		if err := ctrl.SubmitEnvelope(edgeservice.SingleRequestEnvelope{
			RequestID: req.RequestID, Sequence: sequence,
			Stage: edgeservice.SingleRequestStateInternalTool, SavedStage: edgeservice.SingleRequestStatePlanning,
			ToolCall: call,
		}); err != nil {
			return err
		}
		select {
		case result := <-e.results:
			if result.RequestID != req.RequestID || result.StageID != "plan" || result.ToolCallID != call.ToolCallID {
				return errors.New("internal tool result identity mismatch")
			}
		case <-ctx.Done():
			return ctx.Err()
		}
		sequence++
		if err := ctrl.SubmitEnvelope(edgeservice.SingleRequestEnvelope{
			RequestID: req.RequestID, Sequence: sequence,
			Stage: edgeservice.SingleRequestStatePlanning, SavedStage: edgeservice.SingleRequestStatePlanning,
		}); err != nil {
			return err
		}
	}
	for _, stage := range []edgeservice.SingleRequestState{
		edgeservice.SingleRequestStateWorking,
		edgeservice.SingleRequestStateReviewing,
		edgeservice.SingleRequestStateFinalizing,
	} {
		sequence++
		envelope := edgeservice.SingleRequestEnvelope{RequestID: req.RequestID, Sequence: sequence, Stage: stage}
		if stage == edgeservice.SingleRequestStateFinalizing {
			envelope.Result = &edgeservice.SingleRequestResult{Output: "workspace task completed privately"}
		}
		if err := ctrl.SubmitEnvelope(envelope); err != nil {
			return err
		}
	}
	return nil
}

func (e *anthropicInternalToolExecutor) ContinueInternalTool(_ context.Context, result edgeservice.InternalWorkspaceToolResult) error {
	e.continueCount.Add(1)
	e.results <- result.Clone()
	return nil
}

func newAnthropicInternalToolService(
	t *testing.T,
	executor *anthropicInternalToolExecutor,
) (*edgeservice.Service, *toki.TcpClient) {
	t.Helper()
	edgeConn, nodeConn := net.Pipe()
	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, toki.ParserMap{
		toki.TypeNameOf(&iop.WorkspaceOpenResponse{}):    parseAnthropicWorkspaceMessage(&iop.WorkspaceOpenResponse{}),
		toki.TypeNameOf(&iop.WorkspaceToolResponse{}):    parseAnthropicWorkspaceMessage(&iop.WorkspaceToolResponse{}),
		toki.TypeNameOf(&iop.WorkspaceCleanupResponse{}): parseAnthropicWorkspaceMessage(&iop.WorkspaceCleanupResponse{}),
	})
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, toki.ParserMap{
		toki.TypeNameOf(&iop.WorkspaceOpenRequest{}):    parseAnthropicWorkspaceMessage(&iop.WorkspaceOpenRequest{}),
		toki.TypeNameOf(&iop.WorkspaceToolRequest{}):    parseAnthropicWorkspaceMessage(&iop.WorkspaceToolRequest{}),
		toki.TypeNameOf(&iop.WorkspaceCleanupRequest{}): parseAnthropicWorkspaceMessage(&iop.WorkspaceCleanupRequest{}),
	})
	t.Cleanup(func() {
		_ = edgeClient.Close()
		_ = nodeClient.Close()
	})

	registry := edgenode.NewRegistry()
	registry.Register(&edgenode.NodeEntry{NodeID: "workspace-node", Alias: "workspace", Client: edgeClient})
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID: "workspace-node", Alias: "workspace", Token: "workspace-node-token",
		Workspaces: []config.WorkspaceDefinition{{
			Ref: "ws-opaque-ref", Platform: "darwin", Root: "/Users/operator/project",
			Operations:   []config.WorkspaceOperation{config.WorkspaceOpRead, config.WorkspaceOpWrite},
			MaxReadBytes: 4096, MaxWriteBytes: 4096,
		}},
	})
	service := edgeservice.New(registry, nil)
	service.SetNodeStore(store)
	service.SetSingleRequestExecutor(executor)
	return service, nodeClient
}

func parseAnthropicWorkspaceMessage(template proto.Message) func([]byte) (proto.Message, error) {
	return func(payload []byte) (proto.Message, error) {
		message := template.ProtoReflect().Type().New().Interface()
		return message, proto.Unmarshal(payload, message)
	}
}

func TestAnthropicSingleRequestInternalToolsStayPrivate(t *testing.T) {
	executor := newAnthropicInternalToolExecutor()
	service, node := newAnthropicInternalToolService(t, executor)
	var openCount atomic.Int32
	var toolCount atomic.Int32
	toolOrder := make(chan string, 2)
	var cleanupCount atomic.Int32
	toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
		openCount.Add(1)
		return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
	})
	toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
		toolCount.Add(1)
		toolOrder <- req.GetToolCallId()
		response := &iop.WorkspaceToolResponse{
			RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(),
			Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
		}
		if req.GetOperation() == iop.WorkspaceOperation_WORKSPACE_OPERATION_READ {
			response.Content = []byte("private read result")
		}
		return response, nil
	})
	toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
		cleanupCount.Add(1)
		return &iop.WorkspaceCleanupResponse{
			RequestId: req.GetRequestId(),
			Status:    iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
		}, nil
	})

	srv := newAnthropicSingleRequestServer(t, service)
	var httpRequests atomic.Int32
	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		httpRequests.Add(1)
		srv.routes().ServeHTTP(w, r)
	}))
	defer httpServer.Close()

	before := testutil.ToFloat64(singleRequestIngressTotal)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":128,"messages":[{"role":"user","content":"complete the task"}]}`
	request := newAnthropicSingleRequestHTTPReq(t, ctx, httpServer.URL, "/v1/messages", body)
	response, err := httpServer.Client().Do(request)
	if err != nil {
		t.Fatalf("POST /v1/messages: %v", err)
	}
	defer response.Body.Close()
	payload, err := io.ReadAll(response.Body)
	if err != nil {
		t.Fatal(err)
	}
	if response.StatusCode != http.StatusOK {
		t.Fatalf("status=%d body=%s", response.StatusCode, payload)
	}
	var terminal anthropicMessageResponse
	decoder := json.NewDecoder(strings.NewReader(string(payload)))
	if err := decoder.Decode(&terminal); err != nil {
		t.Fatalf("decode terminal: %v", err)
	}
	var extra json.RawMessage
	if err := decoder.Decode(&extra); err != io.EOF {
		t.Fatalf("terminal had trailing output: %v %s", err, extra)
	}
	if len(terminal.Content) != 1 || terminal.Content[0]["text"] != "workspace task completed privately" {
		t.Fatalf("terminal content = %+v", terminal.Content)
	}
	for _, private := range []string{
		"tool_use", "tool_result", edgeservice.InternalWorkspaceToolRead,
		edgeservice.InternalWorkspaceToolWrite, "PRIVATE_INTERNAL_ARGUMENT_SENTINEL", "private read result",
	} {
		if strings.Contains(string(payload), private) {
			t.Fatalf("public terminal leaked %q: %s", private, payload)
		}
	}
	if httpRequests.Load() != 1 || testutil.ToFloat64(singleRequestIngressTotal)-before != 1 {
		t.Fatalf("HTTP requests=%d ingress delta=%v, want 1/1", httpRequests.Load(), testutil.ToFloat64(singleRequestIngressTotal)-before)
	}
	if openCount.Load() != 1 || toolCount.Load() != 2 || cleanupCount.Load() != 1 || executor.continueCount.Load() != 2 {
		t.Fatalf("open=%d tools=%d cleanup=%d continuations=%d, want 1/2/1/2", openCount.Load(), toolCount.Load(), cleanupCount.Load(), executor.continueCount.Load())
	}
	for index, want := range []string{"tool-read", "tool-write"} {
		if got := <-toolOrder; got != want {
			t.Fatalf("tool order[%d]=%q, want %q", index, got, want)
		}
	}
}

type singleRequestMetricKey struct {
	eventClass string
	stage      string
	operation  string
	outcome    string
	errorClass string
}

func singleRequestMetricKeyFromLabels(labels []*dto.LabelPair) (singleRequestMetricKey, error) {
	var key singleRequestMetricKey
	seen := make(map[string]bool)
	for _, lp := range labels {
		name := lp.GetName()
		if seen[name] {
			return singleRequestMetricKey{}, fmt.Errorf("duplicate metric label %q", name)
		}
		seen[name] = true
		switch name {
		case "event_class":
			key.eventClass = lp.GetValue()
		case "stage":
			key.stage = lp.GetValue()
		case "operation":
			key.operation = lp.GetValue()
		case "outcome":
			key.outcome = lp.GetValue()
		case "error_class":
			key.errorClass = lp.GetValue()
		default:
			return singleRequestMetricKey{}, fmt.Errorf("unexpected metric label %q", name)
		}
	}
	expectedLabels := []string{"event_class", "stage", "operation", "outcome", "error_class"}
	for _, expected := range expectedLabels {
		if !seen[expected] {
			return singleRequestMetricKey{}, fmt.Errorf("missing metric label %q", expected)
		}
	}
	return key, nil
}

func isValidSingleRequestCorrelationID(corr string) bool {
	if strings.HasPrefix(corr, "sr-fallback-") {
		rest := corr[len("sr-fallback-"):]
		if len(rest) == 0 || len(rest) > 32 {
			return false
		}
		for _, r := range rest {
			if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'z')) {
				return false
			}
		}
		return true
	}
	if strings.HasPrefix(corr, "sr-") {
		rest := corr[len("sr-"):]
		if len(rest) != 32 {
			return false
		}
		for _, r := range rest {
			if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) {
				return false
			}
		}
		return true
	}
	return false
}

func zapFieldTypeName(t zapcore.FieldType) string {
	switch t {
	case zapcore.StringType:
		return "StringType"
	case zapcore.Int64Type:
		return "Int64Type"
	case zapcore.Int32Type:
		return "Int32Type"
	case zapcore.Float64Type:
		return "Float64Type"
	case zapcore.BoolType:
		return "BoolType"
	default:
		return fmt.Sprintf("FieldType(%d)", t)
	}
}

func singleRequestLogKey(fields []zapcore.Field) (singleRequestMetricKey, string, error) {
	if len(fields) != 9 {
		return singleRequestMetricKey{}, "", fmt.Errorf("expected 9 context fields, got %d", len(fields))
	}
	expectedTypes := map[string]zapcore.FieldType{
		"correlation": zapcore.StringType,
		"event_class": zapcore.StringType,
		"stage":       zapcore.StringType,
		"operation":   zapcore.StringType,
		"outcome":     zapcore.StringType,
		"error_class": zapcore.StringType,
		"duration_ms": zapcore.Int64Type,
		"tool_count":  zapcore.Int64Type,
		"has_result":  zapcore.BoolType,
	}

	seen := make(map[string]bool, len(fields))
	stringVals := make(map[string]string)

	for _, f := range fields {
		if seen[f.Key] {
			return singleRequestMetricKey{}, "", fmt.Errorf("duplicate context key %q", f.Key)
		}
		wantType, ok := expectedTypes[f.Key]
		if !ok {
			return singleRequestMetricKey{}, "", fmt.Errorf("unexpected context key %q", f.Key)
		}
		if f.Type != wantType {
			return singleRequestMetricKey{}, "", fmt.Errorf("key %q has Zap type %s, want %s", f.Key, zapFieldTypeName(f.Type), zapFieldTypeName(wantType))
		}
		seen[f.Key] = true
		if wantType == zapcore.StringType {
			stringVals[f.Key] = f.String
		}
	}

	for k := range expectedTypes {
		if !seen[k] {
			return singleRequestMetricKey{}, "", fmt.Errorf("missing context key %q", k)
		}
	}

	corrVal := stringVals["correlation"]
	if !isValidSingleRequestCorrelationID(corrVal) {
		return singleRequestMetricKey{}, "", fmt.Errorf("invalid correlation format %q", corrVal)
	}

	key := singleRequestMetricKey{
		eventClass: stringVals["event_class"],
		stage:      stringVals["stage"],
		operation:  stringVals["operation"],
		outcome:    stringVals["outcome"],
		errorClass: stringVals["error_class"],
	}
	return key, corrVal, nil
}

func snapshotSingleRequestMetrics(gatherer prometheus.Gatherer) (map[singleRequestMetricKey]float64, map[singleRequestMetricKey]uint64, error) {
	var families []*dto.MetricFamily
	var err error
	families, err = gatherer.Gather()
	if err != nil {
		return nil, nil, err
	}
	counters := make(map[singleRequestMetricKey]float64)
	histograms := make(map[singleRequestMetricKey]uint64)
	for _, family := range families {
		switch family.GetName() {
		case "iop_edge_single_request_lifecycle_total":
			for _, m := range family.GetMetric() {
				key, err := singleRequestMetricKeyFromLabels(m.GetLabel())
				if err != nil {
					return nil, nil, fmt.Errorf("family %s metric key error: %w", family.GetName(), err)
				}
				counters[key] = m.GetCounter().GetValue()
			}
		case "iop_edge_single_request_duration_seconds":
			for _, m := range family.GetMetric() {
				key, err := singleRequestMetricKeyFromLabels(m.GetLabel())
				if err != nil {
					return nil, nil, fmt.Errorf("family %s metric key error: %w", family.GetName(), err)
				}
				histograms[key] = m.GetHistogram().GetSampleCount()
			}
		}
	}
	return counters, histograms, nil
}

// TestAnthropicSingleRequestObservation links ingress, request-total, terminal,
// stage/tool/cleanup counts, and raw-free correlation for a real marked POST
// that exercises deterministic internal tools. It asserts the single-request
// lifecycle produces exactly one accepted ingress, one executor call, one
// terminal acknowledgement, the expected stage/tool/cleanup deltas, and a
// public terminal that never carries internal tool protocol or raw values.
// External Claude/Mac timing evidence is explicitly deferred to claude-smoke.
func TestAnthropicSingleRequestObservation(t *testing.T) {
	executor := newAnthropicInternalToolExecutor()
	service, node := newAnthropicInternalToolService(t, executor)
	var openCount atomic.Int32
	var toolCount atomic.Int32
	var cleanupCount atomic.Int32
	toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
		openCount.Add(1)
		return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
	})
	toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
		toolCount.Add(1)
		response := &iop.WorkspaceToolResponse{
			RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(),
			Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
		}
		return response, nil
	})
	toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
		cleanupCount.Add(1)
		return &iop.WorkspaceCleanupResponse{
			RequestId: req.GetRequestId(),
			Status:    iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS,
		}, nil
	})

	core, logs := observer.New(zap.InfoLevel)
	obsLogger := zap.New(core)
	service.SetSingleRequestObservationLogger(obsLogger)

	srv := newAnthropicSingleRequestServer(t, service)
	httpServer := httptest.NewServer(srv.routes())
	defer httpServer.Close()

	beforeIngress := testutil.ToFloat64(singleRequestIngressTotal)
	beforeCounters, beforeHistograms, err := snapshotSingleRequestMetrics(prometheus.DefaultGatherer)
	if err != nil {
		t.Fatalf("snapshot initial metrics: %v", err)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":128,"messages":[{"role":"user","content":"complete the task"}]}`
	request := newAnthropicSingleRequestHTTPReq(t, ctx, httpServer.URL, "/v1/messages", body)
	response, err := httpServer.Client().Do(request)
	if err != nil {
		t.Fatalf("POST /v1/messages: %v", err)
	}
	defer response.Body.Close()
	payload, err := io.ReadAll(response.Body)
	if err != nil {
		t.Fatal(err)
	}
	if response.StatusCode != http.StatusOK {
		t.Fatalf("status=%d body=%s", response.StatusCode, payload)
	}
	var terminal anthropicMessageResponse
	decoder := json.NewDecoder(strings.NewReader(string(payload)))
	if err := decoder.Decode(&terminal); err != nil {
		t.Fatalf("decode terminal: %v", err)
	}
	var extra json.RawMessage
	if err := decoder.Decode(&extra); err != io.EOF {
		t.Fatalf("terminal had trailing output: %v %s", err, extra)
	}
	if terminal.Model != testSingleRequestModel || terminal.Type != "message" || terminal.Role != "assistant" {
		t.Fatalf("public terminal identity mismatch: %+v", terminal)
	}
	if terminal.StopReason == nil || *terminal.StopReason != "end_turn" {
		t.Fatalf("stop_reason=%v, want end_turn", terminal.StopReason)
	}
	if len(terminal.Content) != 1 || terminal.Content[0]["type"] != "text" {
		t.Fatalf("terminal content=%+v, want one sanitized text block", terminal.Content)
	}
	encoded, err := json.Marshal(terminal)
	if err != nil {
		t.Fatal(err)
	}
	for _, privateValue := range []string{
		"tool_use", "tool_result", edgeservice.InternalWorkspaceToolRead,
		edgeservice.InternalWorkspaceToolWrite, "PRIVATE_INTERNAL_ARGUMENT_SENTINEL",
		"private read result", "ws-opaque-ref", "plan-model", "provider-plan", "slot-plan",
	} {
		if strings.Contains(string(encoded), privateValue) {
			t.Fatalf("terminal leaked %q: %s", privateValue, encoded)
		}
	}
	if got := testutil.ToFloat64(singleRequestIngressTotal) - beforeIngress; got != 1 {
		t.Fatalf("single-request ingress counter delta=%v, want 1", got)
	}
	if got := executor.continueCount.Load(); got != 2 {
		t.Fatalf("executor continuations=%d, want 2 (one per internal tool)", got)
	}
	if openCount.Load() != 1 {
		t.Fatalf("workspace open count=%d, want 1", openCount.Load())
	}
	if toolCount.Load() != 2 {
		t.Fatalf("internal tool count=%d, want 2", toolCount.Load())
	}
	if cleanupCount.Load() != 1 {
		t.Fatalf("workspace cleanup count=%d, want 1", cleanupCount.Load())
	}

	afterCounters, afterHistograms, err := snapshotSingleRequestMetrics(prometheus.DefaultGatherer)
	if err != nil {
		t.Fatalf("snapshot final metrics: %v", err)
	}

	wantDeltas := map[singleRequestMetricKey]float64{
		{eventClass: "request", stage: "none", operation: "total", outcome: "success", errorClass: "none"}:     1,
		{eventClass: "stage", stage: "plan", operation: "plan", outcome: "success", errorClass: "none"}:        1,
		{eventClass: "stage", stage: "work", operation: "work", outcome: "success", errorClass: "none"}:        1,
		{eventClass: "stage", stage: "review", operation: "review", outcome: "success", errorClass: "none"}:    1,
		{eventClass: "tool", stage: "none", operation: "tool", outcome: "success", errorClass: "none"}:         2,
		{eventClass: "cleanup", stage: "none", operation: "cleanup", outcome: "success", errorClass: "none"}:   1,
		{eventClass: "terminal", stage: "none", operation: "terminal", outcome: "success", errorClass: "none"}: 1,
	}

	for key, wantDelta := range wantDeltas {
		gotCounterDelta := afterCounters[key] - beforeCounters[key]
		if gotCounterDelta != wantDelta {
			t.Fatalf("lifecycle metric counter delta for %+v = %v, want %v", key, gotCounterDelta, wantDelta)
		}
		gotHistDelta := afterHistograms[key] - beforeHistograms[key]
		if float64(gotHistDelta) != wantDelta {
			t.Fatalf("lifecycle metric duration delta for %+v = %v, want %v", key, gotHistDelta, wantDelta)
		}
	}

	for key, afterVal := range afterCounters {
		delta := afterVal - beforeCounters[key]
		if delta > 0 {
			if _, expected := wantDeltas[key]; !expected {
				t.Fatalf("unexpected lifecycle metric delta for key %+v: %v", key, delta)
			}
		}
	}

	entries := logs.All()
	if len(entries) != 8 {
		t.Fatalf("captured observation logs count = %d, want 8", len(entries))
	}

	logCounts := make(map[singleRequestMetricKey]float64)
	var requestCorrelation string
	for i, entry := range entries {
		if entry.Message != "edge_single_request_observation" {
			t.Fatalf("log[%d] message = %q, want edge_single_request_observation", i, entry.Message)
		}
		key, corrVal, err := singleRequestLogKey(entry.Context)
		if err != nil {
			t.Fatalf("log[%d] schema: %v", i, err)
		}
		if i == 0 {
			requestCorrelation = corrVal
		} else if corrVal != requestCorrelation {
			t.Fatalf("log[%d] correlation = %q, want shared correlation %q", i, corrVal, requestCorrelation)
		}
		logCounts[key]++

		rawLog := fmt.Sprintf("%+v", entry.ContextMap())
		for _, privateValue := range []string{
			edgeservice.InternalWorkspaceToolRead, edgeservice.InternalWorkspaceToolWrite,
			"README.md", "result.txt", "PRIVATE_INTERNAL_ARGUMENT_SENTINEL",
			"private read result", "ws-opaque-ref", "complete the task",
			"workspace task completed privately",
		} {
			if strings.Contains(rawLog, privateValue) {
				t.Fatalf("log[%d] leaked private content %q: %s", i, privateValue, rawLog)
			}
		}
	}

	if len(logCounts) != len(wantDeltas) {
		t.Fatalf("captured log unique tuple count = %d, want %d", len(logCounts), len(wantDeltas))
	}
	for key, wantCount := range wantDeltas {
		if got := logCounts[key]; got != wantCount {
			t.Fatalf("captured log count for tuple %+v = %v, want %v", key, got, wantCount)
		}
	}

	families, err := prometheus.DefaultGatherer.Gather()
	if err != nil {
		t.Fatalf("gather metrics: %v", err)
	}
	foundMetric := false
	for _, family := range families {
		if family.GetName() != "iop_anthropic_single_request_ingress_total" {
			continue
		}
		foundMetric = true
		for _, metric := range family.Metric {
			if len(metric.Label) != 0 {
				t.Fatalf("single-request ingress metric has request-derived labels: %+v", metric.Label)
			}
		}
	}
	if !foundMetric {
		t.Fatal("registered single-request ingress metric was not gathered")
	}
}

func TestAnthropicSingleRequestUnavailableFailsClosed(t *testing.T) {
	fake := &providerFakeRunService{poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel)}
	srv := newAnthropicSingleRequestServer(t, fake)
	before := testutil.ToFloat64(singleRequestIngressTotal)
	w := httptest.NewRecorder()
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":32,"messages":[{"role":"user","content":"hello"}]}`
	serveAnthropicSingleRequest(t, srv, context.Background(), "/v1/messages", body, w)
	if w.Code != http.StatusServiceUnavailable {
		t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
	}
	if !strings.Contains(w.Body.String(), "single-request execution is unavailable") {
		t.Fatalf("unexpected unavailable body: %s", w.Body.String())
	}
	if got := fake.poolSubmitCountSnapshot(); got != 0 {
		t.Fatalf("marked request fell back to provider pool: submissions=%d", got)
	}
	if got := testutil.ToFloat64(singleRequestIngressTotal) - before; got != 0 {
		t.Fatalf("unavailable capability changed accepted-ingress counter by %v", got)
	}
}

func TestAnthropicSingleRequestExecutorFailureIsSanitized(t *testing.T) {
	const privateFailure = "PRIVATE_PROVIDER_ROUTE_CREDENTIAL_FAILURE"
	executor := anthropicSingleRequestExecutorFunc(func(
		context.Context,
		edgeservice.SingleRequestRequest,
		edgeservice.SingleRequestController,
	) error {
		return errors.New(privateFailure)
	})
	svc := newAdmittedAnthropicSingleRequestService(t, executor, "ws-opaque-ref")
	srv := newAnthropicSingleRequestServer(t, svc)
	w := httptest.NewRecorder()
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":32,"messages":[{"role":"user","content":"hello"}]}`
	serveAnthropicSingleRequest(t, srv, context.Background(), "/v1/messages", body, w)
	if w.Code != http.StatusBadGateway {
		t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
	}
	if strings.Contains(w.Body.String(), privateFailure) || !strings.Contains(w.Body.String(), "single-request execution failed") {
		t.Fatalf("executor failure was not sanitized: %s", w.Body.String())
	}
}

type singleRequestFailingWriter struct {
	header http.Header
	status int
}

func (w *singleRequestFailingWriter) Header() http.Header {
	if w.header == nil {
		w.header = make(http.Header)
	}
	return w.header
}

func (w *singleRequestFailingWriter) WriteHeader(status int) { w.status = status }

func (w *singleRequestFailingWriter) Write([]byte) (int, error) {
	return 0, errors.New("test response write failure")
}

func TestAnthropicSingleRequestWriteFailureRejectsAcknowledgement(t *testing.T) {
	controllerCh := make(chan edgeservice.SingleRequestController, 1)
	executor := anthropicSingleRequestExecutorFunc(func(
		_ context.Context,
		req edgeservice.SingleRequestRequest,
		ctrl edgeservice.SingleRequestController,
	) error {
		if err := submitAnthropicSingleRequestLifecycle(req, ctrl, "safe final"); err != nil {
			return err
		}
		controllerCh <- ctrl
		return nil
	})
	svc := newAdmittedAnthropicSingleRequestService(t, executor, "ws-opaque-ref")
	srv := newAnthropicSingleRequestServer(t, svc)
	w := &singleRequestFailingWriter{}
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":32,"messages":[{"role":"user","content":"hello"}]}`
	serveAnthropicSingleRequest(t, srv, context.Background(), "/v1/messages", body, w)
	if w.status != http.StatusOK {
		t.Fatalf("write status=%d, want attempted 200 terminal", w.status)
	}
	if got := (<-controllerCh).State(); got != edgeservice.SingleRequestStateFailed {
		t.Fatalf("write-failure acknowledgement state=%s, want failed", got)
	}
}

func TestAnthropicSingleRequestCallerCancellationCancelsExecution(t *testing.T) {
	controllerCh := make(chan edgeservice.SingleRequestController, 1)
	executor := anthropicSingleRequestExecutorFunc(func(
		ctx context.Context,
		_ edgeservice.SingleRequestRequest,
		ctrl edgeservice.SingleRequestController,
	) error {
		controllerCh <- ctrl
		<-ctx.Done()
		return ctx.Err()
	})
	svc := newAdmittedAnthropicSingleRequestService(t, executor, "ws-opaque-ref")
	srv := newAnthropicSingleRequestServer(t, svc)
	ctx, cancel := context.WithCancel(context.Background())
	w := httptest.NewRecorder()
	done := make(chan struct{})
	body := `{"model":"` + testSingleRequestModel + `","max_tokens":32,"messages":[{"role":"user","content":"hello"}]}`
	go func() {
		defer close(done)
		serveAnthropicSingleRequest(t, srv, ctx, "/v1/messages", body, w)
	}()
	controller := <-controllerCh
	cancel()
	select {
	case <-done:
	case <-time.After(2 * time.Second):
		t.Fatal("handler did not return after caller cancellation")
	}
	if got := controller.State(); got != edgeservice.SingleRequestStateCancelled {
		t.Fatalf("caller-cancel state=%s, want cancelled", got)
	}
	if w.Body.Len() != 0 {
		t.Fatalf("caller cancellation wrote a terminal after disconnect: %s", w.Body.String())
	}
}

func TestAnthropicSingleRequestCountTokensBypassesExecution(t *testing.T) {
	fake := &providerFakeRunService{poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel)}
	srv := newAnthropicSingleRequestServer(t, fake)
	before := testutil.ToFloat64(singleRequestIngressTotal)
	w := httptest.NewRecorder()
	body := `{"model":"` + testSingleRequestModel + `","messages":[{"role":"user","content":"count this"}]}`
	serveAnthropicSingleRequest(t, srv, context.Background(), "/v1/messages/count_tokens", body, w)
	if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"input_tokens"`) {
		t.Fatalf("count-tokens status=%d body=%s", w.Code, w.Body.String())
	}
	if got := fake.poolSubmitCountSnapshot(); got != 0 {
		t.Fatalf("local count-tokens used provider pool: submissions=%d", got)
	}
	if got := testutil.ToFloat64(singleRequestIngressTotal) - before; got != 0 {
		t.Fatalf("count-tokens changed Messages ingress counter by %v", got)
	}
}

func TestSingleRequestLogSchemaRejectsDrift(t *testing.T) {
	validFields := func() []zapcore.Field {
		return []zapcore.Field{
			zap.String("correlation", "sr-0123456789abcdef0123456789abcdef"),
			zap.String("event_class", "request"),
			zap.String("stage", "none"),
			zap.String("operation", "total"),
			zap.String("outcome", "success"),
			zap.String("error_class", "none"),
			zap.Int64("duration_ms", 15),
			zap.Int("tool_count", 0),
			zap.Bool("has_result", false),
		}
	}

	key, corr, err := singleRequestLogKey(validFields())
	if err != nil {
		t.Fatalf("valid baseline fields rejected: %v", err)
	}
	if corr != "sr-0123456789abcdef0123456789abcdef" {
		t.Fatalf("correlation = %q, want sr-0123456789abcdef0123456789abcdef", corr)
	}
	wantKey := singleRequestMetricKey{
		eventClass: "request",
		stage:      "none",
		operation:  "total",
		outcome:    "success",
		errorClass: "none",
	}
	if key != wantKey {
		t.Fatalf("key = %+v, want %+v", key, wantKey)
	}

	fallbackFields := validFields()
	fallbackFields[0] = zap.String("correlation", "sr-fallback-1a2b3c")
	if _, _, err := singleRequestLogKey(fallbackFields); err != nil {
		t.Fatalf("valid fallback correlation rejected: %v", err)
	}

	tests := []struct {
		name    string
		mutate  func([]zapcore.Field) []zapcore.Field
		wantErr string
	}{
		{
			name: "duplicate_key",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[1] = zap.String("correlation", "sr-0123456789abcdef0123456789abcdef")
				return res
			},
			wantErr: "duplicate context key \"correlation\"",
		},
		{
			name: "missing_key",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				return f[:len(f)-1]
			},
			wantErr: "expected 9 context fields, got 8",
		},
		{
			name: "unknown_key",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[0] = zap.String("unexpected_key", "sr-0123456789abcdef0123456789abcdef")
				return res
			},
			wantErr: "unexpected context key \"unexpected_key\"",
		},
		{
			name: "wrong_numeric_type_float64",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[6] = zap.Float64("duration_ms", 15.0)
				return res
			},
			wantErr: "key \"duration_ms\" has Zap type Float64Type, want Int64Type",
		},
		{
			name: "wrong_numeric_type_int32",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[7] = zap.Int32("tool_count", 0)
				return res
			},
			wantErr: "key \"tool_count\" has Zap type Int32Type, want Int64Type",
		},
		{
			name: "wrong_string_type",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[2] = zap.Int64("stage", 1)
				return res
			},
			wantErr: "key \"stage\" has Zap type Int64Type, want StringType",
		},
		{
			name: "wrong_bool_type",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[8] = zap.String("has_result", "false")
				return res
			},
			wantErr: "key \"has_result\" has Zap type StringType, want BoolType",
		},
		{
			name: "constant_correlation",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[0] = zap.String("correlation", "constant-correlation-id")
				return res
			},
			wantErr: "invalid correlation format \"constant-correlation-id\"",
		},
		{
			name: "short_hex_correlation",
			mutate: func(f []zapcore.Field) []zapcore.Field {
				res := append([]zapcore.Field(nil), f...)
				res[0] = zap.String("correlation", "sr-12345")
				return res
			},
			wantErr: "invalid correlation format \"sr-12345\"",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mutated := tt.mutate(validFields())
			_, _, err := singleRequestLogKey(mutated)
			if err == nil {
				t.Fatalf("expected error containing %q, got nil", tt.wantErr)
			}
			if !strings.Contains(err.Error(), tt.wantErr) {
				t.Fatalf("err = %q, want error containing %q", err.Error(), tt.wantErr)
			}
		})
	}
}